Align released runtime installer pins

This reverts commit 848a4bea10.

.claude/settings.json

@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}

.claude/settings.local.json

@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}

.dockerignore

@@ -3,18 +3,33 @@
 /.heartbeat.json
 /package.json
 /package-lock.json
+__pycache__
+**/__pycache__
 *.pyc
-__pycache__/
+*.pyo
+*.py[cod]
+*$py.class
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
+.uv-cache/
+.cache/
+.cache*/
 .github/
 .pdm-build/
+.pdm-build/**
 .pdm-python
 .eggs/
+*.egg-info/
 .git/
+!.git/
+.git/*
 .vscode/
 !.git/HEAD
-!.git/refs/heads/*
+!.git/packed-refs
+!.git/refs/
+!.git/refs/heads/
+!.git/refs/heads/**
 
 venv/
 .venv/
@@ -31,6 +46,7 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
+tests/out/
 users/
 archive/
 crawls/
@@ -38,6 +54,8 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
+docker-test/
+docker-test*/
 core
 *.core
 
@@ -50,6 +68,8 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
+cache/
+caches/
 assets/
 docker/
 website/

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/SECURITY.md

@@ -2,18 +2,15 @@
 
 ---
 
-## Security Information
+## Security Documentation
 
-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
 
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
+- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -23,12 +20,49 @@ You can also read these pages for more information about ArchiveBox's internals,
 
 ## Reporting a Vulnerability
 
-We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
+~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
 
-1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
+You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
 
-2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
+---
 
-3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
+## CVEs and Report Credits
 
-You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
+> [!IMPORTANT]
+> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
+> Reporters **will still receive credit in release notes** however.
+
+> [!TIP]
+> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
+
+Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
+
+---
+
+## Threat Model
+
+### All admin users are assumed to be trusted
+
+ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
+
+- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
+- non-logged in anonymous guest users
+
+Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
+
+### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
+
+Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
+
+Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
+
+### XSS, CSRF, CORS, CSP
+
+`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
+
+[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
+
+### SSRF / URL Validation
+
+Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
+It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.

.github/configs/ci-linux-build.json

@@ -0,0 +1,81 @@
+{
+  "properties": {
+    "CC_BINARY": {
+      "default": "cc"
+    },
+    "LDAPSEARCH_BINARY": {
+      "default": "ldapsearch"
+    },
+    "INITDB_BINARY": {
+      "default": "initdb"
+    },
+    "PG_CTL_BINARY": {
+      "default": "pg_ctl"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{CC_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "build-essential"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{LDAPSEARCH_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "env": {
+          "version": [
+            "ldapsearch",
+            "-VV"
+          ]
+        },
+        "apt": {
+          "install_args": [
+            "ldap-utils",
+            "python3-dev",
+            "python3-setuptools",
+            "libssl-dev",
+            "libldap2-dev",
+            "libsasl2-dev",
+            "zlib1g-dev",
+            "libatomic1"
+          ],
+          "version": [
+            "ldapsearch",
+            "-VV"
+          ]
+        }
+      }
+    }
+  ],
+  "postgres_binaries": [
+    {
+      "name": "{INITDB_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "postgresql"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{PG_CTL_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "postgresql"
+          ]
+        }
+      }
+    }
+  ]
+}

.github/configs/ci-macos-brew.json

@@ -0,0 +1,13 @@
+{
+  "properties": {
+    "BREW_BINARY": {
+      "default": "brew"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{BREW_BINARY}",
+      "binproviders": "env"
+    }
+  ]
+}

.github/configs/ci-macos-build.json

@@ -0,0 +1,44 @@
+{
+  "properties": {
+    "CLANG_BINARY": {
+      "default": "clang"
+    },
+    "LDAPVC_BINARY": {
+      "default": "ldapvc"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{CLANG_BINARY}",
+      "binproviders": "env,brew",
+      "overrides": {
+        "brew": {
+          "install_args": [
+            "llvm"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{LDAPVC_BINARY}",
+      "binproviders": "env,brew",
+      "overrides": {
+        "env": {
+          "version": [
+            "ldapvc",
+            "-VV"
+          ]
+        },
+        "brew": {
+          "install_args": [
+            "openldap"
+          ],
+          "version": [
+            "ldapvc",
+            "-VV"
+          ]
+        }
+      }
+    }
+  ]
+}

.github/configs/ci-macos-openldap-env.json

@@ -0,0 +1,21 @@
+{
+  "properties": {
+    "LDAPVC_BINARY": {
+      "default": "ldapvc"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{LDAPVC_BINARY}",
+      "binproviders": "env",
+      "overrides": {
+        "env": {
+          "version": [
+            "ldapvc",
+            "-VV"
+          ]
+        }
+      }
+    }
+  ]
+}

.github/configs/ci-tooling.json

@@ -0,0 +1,345 @@
+{
+  "properties": {
+    "CURL_BINARY": {
+      "default": "curl"
+    },
+    "BASH_BINARY": {
+      "default": "bash"
+    },
+    "CHMOD_BINARY": {
+      "default": "chmod"
+    },
+    "DOCKER_BINARY": {
+      "default": "docker"
+    },
+    "GIT_BINARY": {
+      "default": "git"
+    },
+    "GREP_BINARY": {
+      "default": "grep"
+    },
+    "GH_BINARY": {
+      "default": "gh"
+    },
+    "EXPORTFS_BINARY": {
+      "default": "exportfs"
+    },
+    "JQ_BINARY": {
+      "default": "jq"
+    },
+    "MKDIR_BINARY": {
+      "default": "mkdir"
+    },
+    "MOUNT_CIFS_BINARY": {
+      "default": "mount.cifs"
+    },
+    "MOUNT_NFS_BINARY": {
+      "default": "mount.nfs"
+    },
+    "NODE_BINARY": {
+      "default": "node"
+    },
+    "PS_BINARY": {
+      "default": "ps"
+    },
+    "PYTHON_BINARY": {
+      "default": "python"
+    },
+    "PV_BINARY": {
+      "default": "pv"
+    },
+    "SSH_BINARY": {
+      "default": "ssh"
+    },
+    "SED_BINARY": {
+      "default": "sed"
+    },
+    "SMBD_BINARY": {
+      "default": "smbd"
+    },
+    "SUDO_BINARY": {
+      "default": "sudo"
+    },
+    "SYSTEMCTL_BINARY": {
+      "default": "systemctl"
+    },
+    "TAR_BINARY": {
+      "default": "tar"
+    },
+    "TREE_BINARY": {
+      "default": "tree"
+    },
+    "TR_BINARY": {
+      "default": "tr"
+    },
+    "RM_BINARY": {
+      "default": "rm"
+    },
+    "UMOUNT_BINARY": {
+      "default": "umount"
+    },
+    "UV_BINARY": {
+      "default": "uv"
+    },
+    "UNAME_BINARY": {
+      "default": "uname"
+    }
+  },
+  "ci_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{JQ_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{UV_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "bsd_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,pyinfra"
+    }
+  ],
+  "freebsd_binaries": [
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env,pyinfra"
+    }
+  ],
+  "docker_binaries": [
+    {
+      "name": "{DOCKER_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{UNAME_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "deploy_binaries": [
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SSH_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "docker_debug_binaries": [
+    {
+      "name": "{DOCKER_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{PV_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{TAR_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{TREE_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ],
+  "docker_validation_binaries": [
+    {
+      "name": "{DOCKER_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{GREP_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{MKDIR_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{RM_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SED_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SSH_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{TR_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{UNAME_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "mount_validation_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{CHMOD_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{EXPORTFS_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "nfs-kernel-server"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{MKDIR_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{MOUNT_CIFS_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "cifs-utils"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{MOUNT_NFS_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "nfs-common"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{SMBD_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "samba"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SYSTEMCTL_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{UMOUNT_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "process_binaries": [
+    {
+      "name": "{PS_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "docs_binaries": [
+    {
+      "name": "{CURL_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "git_binaries": [
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ],
+  "github_binaries": [
+    {
+      "name": "{GH_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ],
+  "node_binaries": [
+    {
+      "name": "{NODE_BINARY}",
+      "binproviders": "env,npm",
+      "min_version": "22.23.1",
+      "overrides": {
+        "npm": {
+          "install_args": [
+            "node@22.23.1"
+          ],
+          "postinstall_scripts": true
+        }
+      }
+    }
+  ],
+  "release_binaries": [
+    {
+      "name": "{UV_BINARY}",
+      "binproviders": "env,pip,brew"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{GH_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{JQ_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{CURL_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ]
+}

.github/scripts/discover_test_matrix.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Discover every ArchiveBox test file and expose it as one CI job."""
+
+import json
+from pathlib import Path
+
+
+def main() -> None:
+    root = Path.cwd().resolve()
+    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
+
+    if not archivebox_tests:
+        raise SystemExit("No ArchiveBox tests discovered")
+
+    matrix: list[dict[str, object]] = []
+    for path in archivebox_tests:
+        test_path = path.relative_to(root).as_posix()
+        matrix.append(
+            {
+                "name": f"main/{path.stem}",
+                "paths": [test_path],
+                "paths_arg": test_path,
+                "extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
+                "count": 1,
+            },
+        )
+
+    discovered_paths = [path for entry in matrix for path in entry["paths"]]
+    if len(discovered_paths) != len(set(discovered_paths)):
+        raise SystemExit("Tests were not discovered exactly once")
+    if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
+        raise SystemExit("Discovered test shard coverage does not match test files")
+
+    print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
+    print(json.dumps(matrix, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/ci.yml

@@ -0,0 +1,92 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [dev, main]
+  push:
+    branches: [dev, main]
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+  packages: write
+  security-events: write
+
+concurrency:
+  group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
+
+jobs:
+  lint:
+    name: Linters
+    uses: ./.github/workflows/lint.yml
+    secrets: inherit
+
+  install-cli-platform:
+    name: Install and CLI platform compatibility
+    uses: ./.github/workflows/test.yml
+    secrets: inherit
+
+  tests:
+    name: Discovered test matrix
+    uses: ./.github/workflows/test-parallel.yml
+    secrets: inherit
+
+  documentation:
+    name: Documentation and root tests
+    uses: ./.github/workflows/docs.yml
+    secrets: inherit
+
+  codeql:
+    name: CodeQL
+    uses: ./.github/workflows/codeql.yml
+    secrets: inherit
+
+  python-artifacts:
+    name: Tested Python artifacts
+    uses: ./.github/workflows/pip.yml
+    secrets: inherit
+
+  docker-digests:
+    name: Tested Docker images
+    uses: ./.github/workflows/docker.yml
+    with:
+      push_digests: ${{ github.event_name == 'push' }}
+    secrets: inherit
+
+  required:
+    name: All required CI lanes
+    if: ${{ always() }}
+    needs:
+      - lint
+      - install-cli-platform
+      - tests
+      - documentation
+      - codeql
+      - python-artifacts
+      - docker-digests
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Verify every required lane succeeded
+        env:
+          LINT_RESULT: ${{ needs.lint.result }}
+          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
+          TESTS_RESULT: ${{ needs.tests.result }}
+          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
+          CODEQL_RESULT: ${{ needs.codeql.result }}
+          PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
+          DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
+        run: |
+          for result in \
+            "$LINT_RESULT" \
+            "$INSTALL_CLI_PLATFORM_RESULT" \
+            "$TESTS_RESULT" \
+            "$DOCUMENTATION_RESULT" \
+            "$CODEQL_RESULT" \
+            "$PYTHON_ARTIFACTS_RESULT" \
+            "$DOCKER_DIGESTS_RESULT"
+          do
+            [[ "$result" == success ]]
+          done
+          echo 'All required CI lanes and tested release artifacts succeeded.'

.github/workflows/claude.yml

@@ -17,33 +17,24 @@ jobs:
       (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
       (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
       (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       pull-requests: write
       issues: write
       id-token: write
-      actions: read # Required for Claude to read CI results on PRs
+      actions: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 1
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@v1
+        uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-
-          # This is an optional setting that allows Claude to read CI results on PRs
           additional_permissions: |
             actions: read
-
-          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
-          # prompt: 'Update the pull request description to include a summary of changes.'
-
-          # Optional: Add claude_args to customize behavior and configuration
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
           claude_args: '--allowed-tools Bash(gh pr:*)'

.github/workflows/codeql.yml

@@ -1,46 +1,21 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ "dev" ]
-  pull_request:
-    branches: [ "dev" ]
+  workflow_call:
   schedule:
     - cron: '33 17 * * 6'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})
-    # Runner size impacts CodeQL analysis time. To learn more, please see:
-    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
-    #   - https://gh.io/supported-runners-and-hardware-resources
-    #   - https://gh.io/using-larger-runners (GitHub.com only)
-    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
-    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
-    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    runs-on: ubuntu-24.04
     permissions:
-      # required for all workflows
       security-events: write
-
-      # required to fetch internal or private CodeQL packs
       packages: read
-
-      # only required for workflows in private repositories
       actions: read
       contents: read
 
@@ -50,47 +25,18 @@ jobs:
         include:
         - language: python
           build-mode: none
-        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
-        # Use `c-cpp` to analyze code written in C, C++ or both
-        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
-        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
-        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
-        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
-        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
-        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+        - language: javascript-typescript
+          build-mode: none
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-    # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-    # If the analyze step fails for one of the languages you are analyzing with
-    # "We were unable to automatically build your code", modify the matrix above
-    # to set the build mode to "manual" for that language. Then modify this step
-    # to build your code.
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-    - if: matrix.build-mode == 'manual'
-      run: |
-        echo 'If you are using a "manual" build mode for one or more of the' \
-          'languages you are analyzing, replace this with the commands to build' \
-          'your code, for example:'
-        echo '  make bootstrap'
-        echo '  make release'
-        exit 1
-
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/debian.yml

@@ -1,214 +0,0 @@
-name: Build Debian package
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'pkg/debian/**'
-      - 'bin/build_deb.sh'
-      - 'bin/release_deb.sh'
-      - '.github/workflows/debian.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Install nfpm
-        env:
-          NFPM_VERSION: "2.45.1"
-        run: |
-          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
-            | sudo tar -xz -C /usr/local/bin nfpm
-          nfpm --version
-
-      - name: Build .deb package
-        run: |
-          export VERSION="${{ steps.version.outputs.version }}"
-          export ARCH="${{ matrix.arch }}"
-          ./bin/build_deb.sh
-
-      - name: Verify .deb package contents
-        run: |
-          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
-          echo "=== Package info ==="
-          dpkg-deb --info "$DEB_FILE"
-          echo ""
-          echo "=== Package contents ==="
-          dpkg-deb --contents "$DEB_FILE"
-          echo ""
-          echo "=== Control fields ==="
-          dpkg-deb --field "$DEB_FILE"
-
-      - name: Upload .deb artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox-${{ steps.version.outputs.version }}-${{ matrix.arch }}.deb
-          path: dist/*.deb
-
-  test:
-    needs: build
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            runner: ubuntu-24.04
-          - arch: arm64
-            runner: ubuntu-24.04-arm
-    runs-on: ${{ matrix.runner }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local wheel
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --wheel --out-dir /tmp/wheels/
-
-      - name: Download .deb artifact
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*-${{ matrix.arch }}.deb
-          merge-multiple: true
-
-      - name: Install system dependencies
-        run: |
-          sudo apt-get install -y python3.13 python3.13-venv python3-pip git curl wget
-
-      - name: Pre-seed virtualenv with local wheel before dpkg install
-        run: |
-          # CI-only: pre-seed the venv with a local wheel so we test the
-          # unreleased code, not whatever is on PyPI. We explicitly use python3.13
-          # here (matching the system dep installed above) to ensure the venv is
-          # created with the correct Python version. On real installs, install.sh
-          # handles this by preferring python3.13 and failing if < 3.13.
-          # When postinstall.sh runs during dpkg -i, it finds the venv already
-          # populated and just upgrades deps.
-          sudo mkdir -p /opt/archivebox
-          sudo python3.13 -m venv /opt/archivebox/venv
-          sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools
-          sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl
-          echo "[√] Pre-seeded /opt/archivebox/venv with local wheel"
-
-      - name: Install .deb package
-        run: |
-          # dpkg install will run postinstall which creates the user,
-          # sets up systemd, and tries pip install (which finds it already installed)
-          sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
-
-      - name: Verify archivebox is installed
-        run: |
-          which archivebox
-          archivebox version
-
-      - name: Verify wrapper script works
-        run: |
-          /usr/bin/archivebox version
-          /usr/bin/archivebox --help | head -5
-
-      - name: Test archivebox init as archivebox user
-        run: |
-          # The postinstall should have created the user
-          id archivebox
-          sudo mkdir -p /tmp/archivebox-test
-          sudo chown archivebox:archivebox /tmp/archivebox-test
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init'
-
-      - name: Test archivebox status
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'
-
-      - name: Test archivebox add
-        run: |
-          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add "https://example.com"'
-
-      - name: Verify systemd service file exists
-        run: |
-          test -f /usr/lib/systemd/system/archivebox.service
-          cat /usr/lib/systemd/system/archivebox.service
-
-  # Upload .deb to GitHub Release only after tests pass
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: [build, test]
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: write
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Determine release tag
-        id: tag
-        run: |
-          TAG="${{ github.event.release.tag_name }}"
-          if [ -z "$TAG" ]; then
-            TAG="v$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')"
-            echo "[i] No release tag in event context, using version from pyproject.toml: $TAG"
-          fi
-          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-
-      - name: Download all .deb artifacts
-        uses: actions/download-artifact@v4
-        with:
-          pattern: archivebox-*.deb
-          merge-multiple: true
-
-      - name: Upload .deb to GitHub Release
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          TAG="${{ steps.tag.outputs.tag }}"
-          # Verify the release exists before uploading
-          if ! gh release view "$TAG" >/dev/null 2>&1; then
-            echo "[!] No GitHub Release found for tag $TAG."
-            if [ -n "${{ github.event.release.tag_name }}" ]; then
-              echo "[X] This was triggered by a release event — the release should exist. Failing."
-              exit 1
-            fi
-            echo "[i] Skipping upload (workflow_dispatch without a release)."
-            echo "    Create a release first or trigger via the release event."
-            exit 0
-          fi
-          gh release upload "$TAG" *.deb --clobber

.github/workflows/deploy-publicsite.yml

@@ -6,6 +6,13 @@ on:
       - dev
     paths:
       - publicsite/**
+      - archivebox/**
+      - bin/collect_ui_screenshots.*
+      - bin/generate_ui_screenshot_gallery.py
+      - bin/take_screenshot.js
+      - docs/Screenshots.md
+      - pyproject.toml
+      - uv.lock
       - .github/workflows/deploy-publicsite.yml
   workflow_dispatch:
 
@@ -15,29 +22,42 @@ permissions:
   id-token: write
 
 concurrency:
-  group: github-pages-publicsite
-  cancel-in-progress: true
+  group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
 
 jobs:
   deploy:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 1
 
+      - name: Setup Python and uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false
+
+      - name: Install ArchiveBox and screenshot dependencies
+        run: uv sync --no-cache --frozen
+
+      - name: Build UI screenshot gallery
+        run: bin/collect_ui_screenshots.sh
+        env:
+          BASE_URL: http://archivebox.localhost:8000
+
       - name: Setup Pages
-        uses: actions/configure-pages@v5
+        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
         with:
           path: ./publicsite
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4

.github/workflows/docker.yml

@@ -1,27 +1,23 @@
 name: Build Docker image
 
 on:
-  workflow_dispatch:
   workflow_call:
-  push:
-    branches:
-      - dev
-    tags:
-      - 'v*'
-  # pull_request:
+    inputs:
+      push_digests:
+        description: Push validated per-platform images by digest for the release workflow
+        required: false
+        default: false
+        type: boolean
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
 
 permissions:
+  actions: read
   contents: read
   packages: write
 
-concurrency:
-  group: docker-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   build:
     name: build ${{ matrix.platform }}
@@ -34,22 +30,81 @@ jobs:
             runner: ubuntu-24.04
             cache_scope: docker-amd64
             artifact_name: digest-linux-amd64
+            local_tag: archivebox-ci:amd64
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
             cache_scope: docker-arm64
             artifact_name: digest-linux-arm64
+            local_tag: archivebox-ci:arm64
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-        # with:
-        #   submodules: true
-        #   fetch-depth: 1
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Resolve Docker through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python - <<'PY'
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          matches = [package["version"] for package in packages if package["name"] == "abxpkg"]
+          if len(matches) != 1:
+              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
+          print(matches[0])
+          PY
+          )"
+          test -n "$ABXPKG_VERSION"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_VERSION=$ABXPKG_VERSION" >> "$GITHUB_ENV"
+          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+
+          docker_env="$(
+            uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+              --install \
+              --json \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
+          )"
+          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
+          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
+            <<< "$docker_env" >> "$GITHUB_ENV"
+
+          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
+          UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
+          {
+            echo "DOCKER_BINARY=$docker_binary"
+            echo "JQ_BINARY=$JQ_BINARY"
+            echo "UV_BINARY=$UV_BINARY"
+          } >> "$GITHUB_ENV"
+          for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
+            test -L "$binary"
+            test -x "$binary"
+          done
+          "$docker_binary" --version
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
-          version: latest
           install: true
 
       - name: Builder instance name
@@ -58,15 +113,50 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
+      - name: Verify published abx-dl image
+        id: abx_dl_image
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
+          if len(matches) != 1:
+              raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
+          print(matches[0])
+          PY
+          )"
+          test -n "$ABX_DL_VERSION"
+          ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
+          INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
+          printf '%s\n' "$INSPECT_OUTPUT"
+          ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
+          import os
+
+          digest = next(
+              (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
+              "",
+          )
+          print(digest)
+          PY
+          )"
+          [[ "$ABX_DL_DIGEST" == sha256:* ]]
+          ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
+          echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
+
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        if: inputs.push_digests
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        if: inputs.push_digests
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -77,49 +167,297 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          VERSION="$(python3 - <<'PY'
+          VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
+          import tomllib
           from pathlib import Path
-          import re
 
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
+          print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
           PY
           )"
+          test -n "$VERSION"
 
           {
             echo 'labels<<EOF'
             echo "org.opencontainers.image.version=${VERSION}"
             echo "org.opencontainers.image.revision=${GITHUB_SHA}"
             echo 'org.opencontainers.image.source=https://github.com/ArchiveBox/ArchiveBox'
+            echo "io.archivebox.abx-dl.image=${{ steps.abx_dl_image.outputs.image }}"
             echo 'EOF'
             echo "version=${VERSION}"
           } >> "$GITHUB_OUTPUT"
 
-          echo "[+] Building ${{ matrix.platform }} for ${VERSION}"
+          echo "[+] Building ${{ matrix.platform }} for ${VERSION} using ${{ steps.abx_dl_image.outputs.image }}"
 
       - name: Build and push digest
         id: docker_build
-        uses: docker/build-push-action@v6
+        if: inputs.push_digests
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
         with:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: |
-            ${{ env.DOCKERHUB_IMAGE }}
-            ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
-          cache-from: type=gha,scope=${{ matrix.cache_scope }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+          pull: true
+          platforms: ${{ matrix.platform }}
+          outputs: |
+            type=docker,name=${{ matrix.local_tag }}
+            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
+            type=image,"name=${{ env.DOCKERHUB_IMAGE }},${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true
+
+      - name: Build pull request image
+        if: ${{ !inputs.push_digests }}
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        with:
+          context: ./
+          file: ./Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          load: true
+          tags: ${{ matrix.local_tag }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+          pull: true
           platforms: ${{ matrix.platform }}
-          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
+        if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Validate exact built image version and commit
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          IMAGE="${{ matrix.local_tag }}"
+          SHORT_SHA="${GITHUB_SHA::7}"
+
+          DATA_DIR="$(mktemp -d)"
+          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
+          ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
+          printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
+          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
+          IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
+          ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
+            "$UV_BINARY" run --no-cache --no-project python - \
+            "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
+            "COMMIT_HASH=${SHORT_SHA}" \
+            "COMMIT_HASH=${GITHUB_SHA}" <<'PY'
+          import os
+          import sys
+
+          version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
+          image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
+          if sys.argv[1] not in version_output:
+              raise SystemExit(f"Missing version marker: {sys.argv[1]}")
+          if sys.argv[2] not in version_output:
+              raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
+          if sys.argv[3] not in image_version_lines:
+              raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
+          PY
+
+          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
+            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); projections = [projection for record in records.values() for projection in record.get("request_exec_projections", {}).values()]; plans = [projection["validation"] for projection in projections]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert projections and {plan.get("euid") for plan in plans} == {911}; assert all(not record.get("script_exec_plans") for record in records.values()); assert "ABXPKG_TMP_CACHE_DIR" not in json.dumps(projections); assert not mismatches, mismatches[:3]'
+
+      - name: Resolve real Docker CLI test dependencies through abxpkg
+        if: matrix.platform == 'linux/amd64'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mount_env="$(
+            uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+              --install \
+              --json \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
+          )"
+          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
+            <<< "$mount_env" >> "$GITHUB_ENV"
+
+          for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
+            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
+            test -L "$binary"
+            test -x "$binary"
+          done
+
+      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
+        if: matrix.platform == 'linux/amd64'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          NFS_SOURCE="/srv/archivebox-ci-nfs"
+          SMB_SOURCE="/srv/archivebox-ci-smb"
+          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
+          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
+          mounted_nfs=0
+          mounted_smb=0
+
+          cleanup() {
+            local original_status=$? cleanup_status=0
+            trap - EXIT
+            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
+              cleanup_status=1
+            fi
+            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
+              cleanup_status=1
+            fi
+            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
+              cleanup_status=1
+            fi
+            if [[ "$original_status" != "0" ]]; then
+              exit "$original_status"
+            fi
+            exit "$cleanup_status"
+          }
+          trap cleanup EXIT
+
+          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
+          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+          "$SUDO_BINARY" "$BASH_BINARY" -c \
+            "printf '%s *(rw,sync,no_subtree_check,no_root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
+          "$SUDO_BINARY" "$BASH_BINARY" -c \
+            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
+
+          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
+          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
+          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
+          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
+          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
+
+          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
+          mounted_nfs=1
+          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
+            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
+          mounted_smb=1
+
+          NFS_TEST_DIR="$NFS_MOUNT" \
+          SMB_TEST_DIR="$SMB_MOUNT" \
+          IMAGE="${{ matrix.local_tag }}" \
+            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
+
+      - name: Validate Docker CLI add/list and compose behavior
+        if: matrix.platform == 'linux/amd64'
+        shell: bash
+        env:
+          ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }}
+        run: |
+          set -Eeuo pipefail
+          "$MKDIR_BINARY" "${{ github.workspace }}/data"
+          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
+
+          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
+
+          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
+          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
+
+          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
+          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
+          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
+
+          "$DOCKER_BINARY" compose run --rm archivebox init
+          "$DOCKER_BINARY" compose up -d --wait
+          "$DOCKER_BINARY" network connect archivebox_default www-nginx
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
+          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
+          [[ "$index_html" == *ArchiveBox* ]]
+          [[ "$admin_js" == *window.django* ]]
+
+          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
+          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
+          "$DOCKER_BINARY" compose down
+
+      - name: Validate compressed image size
+        if: inputs.push_digests
+        env:
+          BUILD_METADATA: ${{ steps.docker_build.outputs.metadata }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
+          "$UV_BINARY" run --no-cache --no-project python - \
+            '${{ runner.temp }}/archivebox-image.tar' \
+            '${{ steps.docker_build.outputs.digest }}' \
+            "$LOCAL_IMAGE_ID" \
+            "$((910 * 1024 * 1024))" <<'PY'
+          from hashlib import sha256
+          import json
+          import os
+          import sys
+          import tarfile
+
+          archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
+          limit = int(limit_text)
+          metadata = json.loads(os.environ["BUILD_METADATA"])
+          if metadata["containerimage.digest"] != pushed_digest:
+              raise SystemExit("Build push transaction digest does not match the action output")
+          if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
+              raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
+
+          with tarfile.open(archive_path) as archive:
+              def read_blob(descriptor):
+                  algorithm, digest = descriptor["digest"].split(":", 1)
+                  if algorithm != "sha256":
+                      raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
+                  member = archive.extractfile(f"blobs/sha256/{digest}")
+                  if member is None:
+                      raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
+                  blob = member.read()
+                  if len(blob) != descriptor["size"]:
+                      raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
+                  if sha256(blob).hexdigest() != digest:
+                      raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
+                  return blob
+
+              index_member = archive.extractfile("index.json")
+              if index_member is None:
+                  raise SystemExit("OCI export is missing index.json")
+              index = json.load(index_member)
+
+              manifests = []
+
+              def collect(descriptor):
+                  document = json.loads(read_blob(descriptor))
+                  media_type = descriptor["mediaType"]
+                  if "image.index" in media_type or "manifest.list" in media_type:
+                      for child in document["manifests"]:
+                          collect(child)
+                  elif "image.manifest" in media_type:
+                      config_type = document["config"]["mediaType"]
+                      if "image.config" in config_type or "container.image" in config_type:
+                          manifests.append(document)
+
+              for root in index["manifests"]:
+                  collect(root)
+              if len(manifests) != 1:
+                  raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
+
+              manifest = manifests[0]
+              config_digest = manifest["config"]["digest"]
+              if metadata["containerimage.config.digest"] != config_digest:
+                  raise SystemExit("OCI config digest does not match the pushed build result")
+              if local_image_id != config_digest:
+                  raise SystemExit("Locally tested image does not match the OCI build result")
+              descriptors = [manifest["config"], *manifest["layers"]]
+              for descriptor in descriptors:
+                  read_blob(descriptor)
+              total = sum(descriptor["size"] for descriptor in descriptors)
+
+          print(
+              f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
+              f"limit={limit // 1024 // 1024} MiB",
+          )
+          if total > limit:
+              raise SystemExit(f"{pushed_digest} is over the compressed size limit")
+          PY
+
       - name: Export digest
+        if: inputs.push_digests
         shell: bash
         run: |
           set -Eeuo pipefail
@@ -128,134 +466,10 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@v4
+        if: inputs.push_digests
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: ${{ matrix.artifact_name }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
-
-  publish:
-    name: publish multiarch tags
-    runs-on: ubuntu-24.04
-    needs:
-      - build
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: latest
-          install: true
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-
-      - name: Collect Docker tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(python3 - <<'PY'
-          from pathlib import Path
-          import re
-
-          match = re.search(r'^version = "([^"]+)"$', Path("pyproject.toml").read_text(), re.MULTILINE)
-          if not match:
-              raise SystemExit("Failed to read version from pyproject.toml")
-          print(match.group(1))
-          PY
-          )"
-          SHORT_SHA="${GITHUB_SHA::8}"
-
-          {
-            echo 'dockerhub_tags<<EOF'
-            echo "${DOCKERHUB_IMAGE}:dev"
-            echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            echo "${GHCR_IMAGE}:dev"
-            echo "${GHCR_IMAGE}:${VERSION}"
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            echo 'EOF'
-            echo "version=${VERSION}"
-            echo "short_sha=${SHORT_SHA}"
-          } >> "$GITHUB_OUTPUT"
-
-          echo "[+] Publishing Docker Hub tags:"
-          printf '%s\n' "${DOCKERHUB_IMAGE}:dev" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-          echo "[+] Publishing GHCR tags:"
-          printf '%s\n' "${GHCR_IMAGE}:dev" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
-          [[ "${#DIGESTS[@]}" -gt 0 ]]
-
-          TAG_ARGS=()
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
-          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
-
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do
-            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
-          done
-
-          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
-          [[ "${#DIGESTS[@]}" -gt 0 ]]
-
-          TAG_ARGS=()
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
-          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
-
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do
-            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
-          done
-
-          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Inspect published images
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-
-      - name: Update README
-        uses: peter-evans/dockerhub-description@v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox

.github/workflows/docs.yml

@@ -0,0 +1,156 @@
+name: Documentation tests
+
+on:
+  workflow_call:
+
+env:
+  PYTHONIOENCODING: utf-8
+  USE_COLOR: "False"
+  SHOW_PROGRESS: "False"
+
+jobs:
+  discover:
+    name: Discover documentation examples
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    outputs:
+      matrix: ${{ steps.inventory.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+      - name: Install ArchiveBox development dependencies
+        run: uv sync --no-cache --locked --dev
+      - name: Resolve documentation tools through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
+        run: |
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          uv run --no-cache --no-sync abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
+          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
+          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
+          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
+          test -L "$bash_binary"
+          test -L "$mmdc_binary"
+          test -L "$nginx_binary"
+          test -x "$bash_binary"
+          test -x "$mmdc_binary"
+          test -x "$nginx_binary"
+          {
+            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
+            echo "BASH_BINARY=$bash_binary"
+          } >> "$GITHUB_ENV"
+      - id: inventory
+        name: Validate the complete authored documentation inventory
+        run: |
+          matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+
+  examples:
+    name: docs/${{ matrix.name }}
+    needs: discover
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        with:
+          python-version: "3.13"
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+      - name: Install ArchiveBox
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        run: uv sync --no-cache --locked --dev
+      - name: Resolve documentation runtime through abxpkg
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
+        run: |
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          dependency_sources=(
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
+          )
+          if [[ '${{ matrix.environment }}' != macos ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
+          fi
+          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
+          elif [[ '${{ matrix.environment }}' == docker ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
+          elif [[ '${{ matrix.environment }}' == macos ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
+          fi
+          uv run --no-cache --no-sync abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            "${dependency_sources[@]}"
+          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
+          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
+          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
+          test -L "$bash_binary"
+          test -x "$bash_binary"
+          test -L "$python_binary"
+          test -x "$python_binary"
+          test -L "$uv_binary"
+          test -x "$uv_binary"
+          {
+            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
+            echo "BASH_BINARY=$bash_binary"
+            echo "PYTHON_BINARY=$python_binary"
+            echo "UV_BINARY=$uv_binary"
+          } >> "$GITHUB_ENV"
+          if [[ '${{ matrix.environment }}' != macos ]]; then
+            sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
+            test -L "$sudo_binary"
+            test -x "$sudo_binary"
+            echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
+          fi
+          if [[ '${{ matrix.environment }}' != macos ]]; then
+            echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+          fi
+          if [[ '${{ matrix.environment }}' == docker ]]; then
+            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
+            test -L "$docker_binary"
+            test -x "$docker_binary"
+            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
+          elif [[ '${{ matrix.environment }}' == macos ]]; then
+            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
+            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
+          fi
+      - name: Build the exact checkout's documentation image
+        if: ${{ matrix.environment == 'docker' }}
+        run: '"$DOCKER_BINARY" build --tag archivebox/archivebox:dev .'
+      - name: Run the Ubuntu, Docker, or macOS documentation lane
+        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
+        run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Run the root documentation lane
+        if: ${{ matrix.environment == 'root' }}
+        run: |
+          "$SUDO_BINARY" \
+            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
+            "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'

.github/workflows/duplicate-issue-detection.yml

@@ -6,18 +6,55 @@ on:
 
 jobs:
   check-duplicates:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       issues: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 1
 
-      - name: Install opencode
-        run: curl -fsSL https://opencode.ai/install | bash
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+
+      - name: Resolve OpenCode and workflow tools through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        run: |
+          set -Eeuo pipefail
+          uv sync --no-cache --locked --no-sources --no-install-project
+          OPENCODE_CONFIG="$(uv run --no-cache --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "opencode" / "config.json")')"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+          opencode_env="$(
+            uv run --no-cache --no-sync --no-sources abxpkg env \
+              --install \
+              --json \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:github_binaries" \
+              --deps-from="$OPENCODE_CONFIG:required_binaries"
+          )"
+          jq_binary="$ABXPKG_LIB_DIR/env/bin/jq"
+          "$jq_binary" -r 'to_entries[] | "\(.key)=\(.value)"' \
+            <<< "$opencode_env" >> "$GITHUB_ENV"
+
+          opencode_binary="$ABXPKG_LIB_DIR/env/bin/opencode"
+          echo "OPENCODE_BINARY=$opencode_binary" >> "$GITHUB_ENV"
+          test -L "$opencode_binary"
+          test -x "$opencode_binary"
 
       - name: Check for duplicate issues
         env:
@@ -33,7 +70,7 @@ jobs:
               "webfetch": "allow"
             }
         run: |
-          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+          "$OPENCODE_BINARY" run -m anthropic/claude-haiku-4-5 "A new issue has been created:
 
           Issue number:
           ${{ github.event.issue.number }}

.github/workflows/homebrew.yml

@@ -1,233 +0,0 @@
-name: Build Homebrew formula
-
-on:
-  workflow_dispatch:
-  workflow_call:
-  push:
-    branches: ['**']
-    paths:
-      - 'brew_dist/**'
-      - 'bin/build_brew.sh'
-      - 'bin/release_brew.sh'
-      - '.github/workflows/homebrew.yml'
-      - 'pyproject.toml'
-  # release trigger is handled by release.yml to avoid double-runs
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build-and-test:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-latest, ubuntu-24.04]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Validate formula template syntax
-        run: ruby -c brew_dist/archivebox.rb
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: false
-
-      - name: Install build dependencies (Linux)
-        if: runner.os == 'Linux'
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.6.0
-        with:
-          packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1
-          version: 1.0
-
-      - name: Build local sdist
-        run: |
-          uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-          uv build --sdist --out-dir /tmp/sdist/
-
-      - name: Generate formula from local sdist
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)"
-          SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')"
-
-          # Install archivebox + poet into a temp venv to generate resource stanzas
-          python3 -m venv /tmp/poet-venv
-          source /tmp/poet-venv/bin/activate
-          pip install --quiet "$SDIST_PATH" homebrew-pypi-poet
-          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
-          RESOURCES="$(poet archivebox)"
-          deactivate
-
-          # For CI: use file:// URL pointing to local sdist
-          # For release: this gets overridden with the PyPI URL
-          SDIST_URL="file://${SDIST_PATH}"
-
-          cat > /tmp/archivebox.rb << RUBY
-class Archivebox < Formula
-  include Language::Python::Virtualenv
-
-  desc "Self-hosted internet archiving solution"
-  homepage "https://github.com/ArchiveBox/ArchiveBox"
-  url "${SDIST_URL}"
-  sha256 "${SDIST_SHA256}"
-  license "MIT"
-
-  depends_on "python@3.13"
-
-${RESOURCES}
-
-  def install
-    virtualenv_install_with_resources
-  end
-
-  def post_install
-    data_dir = var/"archivebox"
-    data_dir.mkpath
-    ENV["DATA_DIR"] = data_dir.to_s
-    system bin/"archivebox", "init"
-  end
-
-  def caveats
-    <<~EOS
-      ArchiveBox data is stored in:
-        #{var}/archivebox
-
-      To start archiving, run:
-        cd #{var}/archivebox && archivebox add 'https://example.com'
-
-      To start the web UI:
-        cd #{var}/archivebox && archivebox server 0.0.0.0:8000
-    EOS
-  end
-
-  test do
-    assert_match version.to_s, shell_output("#{bin}/archivebox version")
-  end
-end
-RUBY
-
-          echo "[√] Generated formula:"
-          ruby -c /tmp/archivebox.rb
-          cat /tmp/archivebox.rb
-
-      - name: Install Homebrew (Linux only)
-        if: runner.os == 'Linux'
-        run: |
-          NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV"
-          echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV"
-          echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV"
-          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
-          echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH"
-
-      - name: Install brew dependencies
-        run: brew install python@3.13
-
-      - name: Install archivebox via brew from local formula
-        run: |
-          brew install --build-from-source --verbose /tmp/archivebox.rb
-
-      - name: Verify archivebox version
-        run: archivebox version
-
-      - name: Test archivebox init
-        run: |
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init
-
-      - name: Test archivebox status
-        run: |
-          cd /tmp/archivebox-test
-          archivebox status
-
-  # On release only: generate the real formula with PyPI URL and push to tap
-  release:
-    if: github.event_name == 'release' || github.event_name == 'workflow_call'
-    needs: build-and-test
-    runs-on: macos-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 1
-
-      - name: Get version
-        id: version
-        run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Wait for PyPI package availability
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
-          for i in $(seq 1 30); do
-            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
-              echo "[√] archivebox==${VERSION} is available on PyPI"
-              break
-            fi
-            if [ "$i" -eq 30 ]; then
-              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
-              break
-            fi
-            echo "    Attempt $i/30 - not yet available, waiting 30s..."
-            sleep 30
-          done
-
-      - name: Generate release formula via build_brew.sh
-        run: ./bin/build_brew.sh
-
-      - name: Test formula install
-        run: |
-          brew install --build-from-source brew_dist/archivebox.rb
-          archivebox version
-
-      - name: Upload formula artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: archivebox.rb
-          path: brew_dist/archivebox.rb
-
-      - name: Push to homebrew-archivebox tap
-        env:
-          GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          VERSION="${{ steps.version.outputs.version }}"
-          git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-
-          cp brew_dist/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
-            cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb
-
-          cd /tmp/tap
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add -A
-          if git diff --cached --quiet; then
-            echo "[i] No changes to formula, skipping push."
-          else
-            git commit -m "Update archivebox to v${VERSION}"
-            git push origin HEAD
-            echo "[√] Formula pushed to homebrew-archivebox tap"
-          fi

.github/workflows/lint.yml

@@ -1,50 +1,38 @@
 name: Run linters
 
 on:
-  workflow_dispatch:
-  push:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   UV_NO_SOURCES: "1"
-  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
 jobs:
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: "3.13"
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
+          version: "0.11.3"
           enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Verify checked dependency lock
+        run: uv lock --no-cache --check
 
       - name: Install dependencies with uv
-        run: |
-          uv sync --extra sonic --extra debug --all-groups --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
+        run: uv sync --no-cache --locked --dev --extra sonic --extra debug
 
       - name: Run prek
-        run: uv run --no-sync --no-sources prek run --all-files
+        run: uv run --no-cache --no-sync --no-sources prek run --all-files

.github/workflows/pip.yml

@@ -1,69 +1,221 @@
 name: Build Pip package
 
 on:
-  workflow_dispatch:
   workflow_call:
-  push:
-    branches:
-      - '**'
-    tags:
-      - 'v*'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 
 env:
-  PYTHON_VERSION: "3.13"
+  UV_VERSION: "0.11.3"
 
 jobs:
   build:
-    permissions:
-      id-token: write
-
+    name: build distributions
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
+      - name: Set up Python 3.13
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
-          enable-cache: false
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "3.13"
           architecture: x64
 
-      - name: APT install archivebox dev + run dependencies
-        uses: awalsh128/cache-apt-pkgs-action@latest
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
+          version: ${{ env.UV_VERSION }}
+          enable-cache: false
+          cache-dependency-glob: uv.lock
 
-      - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-
-      - name: UV build archivebox and archivebox/pkgs/* packages
+      - name: Build every distribution
         run: |
-          uv build --all
+          set -Eeuo pipefail
+          uv build --no-cache --all-packages --no-sources --out-dir dist --clear
 
-      - name: Publish new package wheels and sdists to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        # ignore when publish to PyPI fails due to duplicate tag
-        continue-on-error: true
+          shopt -s nullglob
+          artifacts=(dist/*)
+          wheels=(dist/archivebox-*.whl)
+          sdists=(dist/archivebox-*.tar.gz)
+          [[ "${#artifacts[@]}" -gt 0 ]]
+          [[ "${#wheels[@]}" -eq 1 ]]
+          [[ "${#sdists[@]}" -eq 1 ]]
+          printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
+          uv run --no-cache --no-project python - <<'PY'
+          import hashlib
+          from pathlib import Path
+
+          root = Path("dist")
+          wheels = list(root.glob("archivebox-*.whl"))
+          sdists = list(root.glob("archivebox-*.tar.gz"))
+          if len(wheels) != 1 or len(sdists) != 1:
+              raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
+          artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
+          checksum_lines = []
+          for artifact in artifacts:
+              digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
+              line = f"{digest}  {artifact.name}"
+              checksum_lines.append(line)
+              print(line)
+          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
+          PY
+
+      - name: Upload every distribution
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}
+          name: python-distributions
+          path: dist/*
+          if-no-files-found: error
+          retention-days: 7
 
-      - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --all-extras --no-sources --no-cache
+  install-smoke:
+    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, macos-15]
+        python: ["3.13", "3.14"]
+        artifact: [wheel, sdist]
 
-      - name: UV run archivebox init + archivebox version
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: ${{ env.UV_VERSION }}
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Download distributions
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: python-distributions
+          path: ${{ runner.temp }}/python-distributions
+
+      - name: Install and smoke-test the built ${{ matrix.artifact }}
+        shell: bash
         run: |
-          mkdir -p data && cd data
-          uv run --no-sync --no-sources archivebox init \
-          && uv run --no-sync --no-sources archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          set -Eeuo pipefail
+          shopt -s nullglob
+
+          case "${{ matrix.artifact }}" in
+            wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
+            sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
+            *) exit 2 ;;
+          esac
+          [[ "${#candidates[@]}" -eq 1 ]]
+          artifact="${candidates[0]}"
+
+          smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
+          trap 'rm -rf "$smoke_root"' EXIT
+          smoke_env="$smoke_root/env"
+          data_dir="$smoke_root/data"
+          mkdir -p "$data_dir"
+
+          dependency_artifacts=()
+          while IFS= read -r dependency_artifact; do
+            dependency_artifacts+=("$dependency_artifact")
+          done < <(uv run --no-cache --no-project python - <<'PY'
+          import tomllib
+
+          locked = {
+              package["name"]: package
+              for package in tomllib.load(open("uv.lock", "rb"))["package"]
+          }
+          for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
+              package = locked[package_name]
+              wheels = package.get("wheels") or []
+              if len(wheels) != 1:
+                  raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
+              print(wheels[0]["url"])
+          PY
+          )
+
+          uv venv --python "${{ matrix.python }}" "$smoke_env"
+          uv pip install --no-cache \
+            --python "$smoke_env" \
+            "${dependency_artifacts[@]}" \
+            "$artifact"
+
+          unset PYTHONPATH
+          cd "$data_dir"
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
+          import os
+          from pathlib import Path
+
+          import archivebox
+
+          installed_path = Path(archivebox.__file__).resolve()
+          environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
+          workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
+          assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
+          assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
+          print(installed_path)
+          PY
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
+
+          if [[ "${{ runner.os }}" == "Linux" ]]; then
+            root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
+            uv_binary="$(command -v uv)"
+            sudo rm -rf "$root_tool_dir"
+            sudo mkdir -p "$root_tool_dir/data"
+
+            root_install_args=()
+            for dependency_artifact in "${dependency_artifacts[@]}"; do
+              root_install_args+=(--with "$dependency_artifact")
+            done
+            sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
+              "$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
+              "${root_install_args[@]}" "$artifact"
+
+            sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
+            sudo rm -rf /var/lib/archivebox
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data' && archivebox init"
+            sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
+            sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
+            if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
+              echo "Root-owned files remain in $root_tool_dir/data" >&2
+              exit 1
+            fi
+
+            sudo mkdir "$root_tool_dir/external"
+            sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data' && archivebox status"
+            sudo stat -c %U "$root_tool_dir/external" | grep -qx root
+
+            sudo mkdir "$root_tool_dir/data-universal-init"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
+            sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
+
+            sudo mkdir "$root_tool_dir/data-install"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
+            sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
+
+            sudo useradd --create-home archivebox-smoke-user
+            smoke_user_group="$(id -gn archivebox-smoke-user)"
+            sudo mkdir "$root_tool_dir/data-mixed-owner"
+            sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
+            sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
+            if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
+              echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
+              exit 1
+            fi
+            sudo rm -rf "$root_tool_dir"
+          fi

.github/workflows/release-runner.yml

@@ -1,49 +0,0 @@
-name: Release State
-
-on:
-  push:
-    branches:
-      - '**'
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  release-state:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ github.ref_name }}
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Configure git identity
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Run release script
-        env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
-        run: ./bin/release.sh

.github/workflows/release.yml

@@ -1,47 +1,324 @@
 name: Release
 
-# Orchestrates the full release pipeline:
-#   1. Build and publish pip package to PyPI
-#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
-#   3. Build Docker images (in parallel with deb/brew)
-#
-# Individual workflows also run on push for CI (see their own triggers).
-# Manual fallback for the hosted release pipeline. The normal prerelease loop
-# is run by bin/release.sh locally so published GitHub releases do not launch a
-# duplicate release job.
-
 on:
-  workflow_dispatch:
+  workflow_run:
+    workflows: [CI]
+    types: [completed]
+    branches: [dev, main]
 
-permissions:
-  contents: write
-  packages: write
-  id-token: write
+permissions: {}
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  group: release-${{ github.repository }}-${{ github.event.workflow_run.head_branch }}
+  cancel-in-progress: false
+
+env:
+  DOCKERHUB_IMAGE: archivebox/archivebox
+  GHCR_IMAGE: ghcr.io/archivebox/archivebox
+  RELEASE_BRANCH: ${{ github.event.workflow_run.head_branch }}
+  RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
+  CI_RUN_ID: ${{ github.event.workflow_run.id }}
 
 jobs:
-  pip:
-    name: Publish to PyPI
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
+  python-release:
+    name: Publish tested Python artifacts
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push'
+    runs-on: ubuntu-24.04
+    environment: pypi
+    permissions:
+      actions: read
+      contents: write
+      id-token: write
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      tag: ${{ steps.version.outputs.tag }}
+      release_ready: ${{ steps.release-state.outputs.ready }}
+    steps:
+      - name: Checkout the tested commit
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          ref: ${{ env.RELEASE_SHA }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
 
-  debian:
-    name: Build .deb packages
-    needs: pip
-    uses: ./.github/workflows/debian.yml
-    secrets: inherit
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
 
-  homebrew:
-    name: Update Homebrew formula
-    needs: pip
-    uses: ./.github/workflows/homebrew.yml
-    secrets: inherit
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
 
-  docker:
-    name: Build Docker images
-    needs: pip
-    uses: ./.github/workflows/docker.yml
-    secrets: inherit
+      - name: Resolve release binaries through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
+          for name in uv gh git jq curl; do
+            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
+            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
+          done
+          {
+            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
+            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
+            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
+            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
+            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
+          } >> "$GITHUB_ENV"
+
+      - name: Download exact tested distributions
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: python-distributions
+          path: ${{ runner.temp }}/python-distributions
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.CI_RUN_ID }}
+
+      - name: Record release version
+        id: version
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Validate release branch and version
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION='${{ steps.version.outputs.version }}'
+          case "$RELEASE_BRANCH" in
+            main)
+              ;;
+            dev)
+              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
+                echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
+                exit 1
+              }
+              ;;
+            *)
+              echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
+              exit 1
+              ;;
+          esac
+
+      - name: Publish the exact tested distributions
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+          RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
+        run: ./bin/release.sh
+
+      - name: Confirm this tested commit owns the release tag
+        id: release-state
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          TARGET=""
+          while read -r sha _; do
+            TARGET="$sha"
+          done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
+          if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
+            echo "ready=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "ready=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Verify published PyPI package installs and runs
+        if: steps.release-state.outputs.ready == 'true'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION='${{ steps.version.outputs.version }}'
+          TOOL_ROOT="$(mktemp -d)"
+          export UV_TOOL_DIR="$TOOL_ROOT/tools"
+          export UV_TOOL_BIN_DIR="$TOOL_ROOT/bin"
+          for attempt in {1..40}; do
+            if "$UV_BINARY" tool install --no-cache --force "archivebox==$VERSION"; then
+              break
+            fi
+            if [[ "$attempt" == 40 ]]; then
+              echo "archivebox==$VERSION did not become installable from PyPI within 10 minutes" >&2
+              exit 1
+            fi
+            sleep 15
+          done
+          INSTALLED_VERSION="$("$UV_TOOL_BIN_DIR/archivebox" --version)"
+          test "$INSTALLED_VERSION" = "$VERSION"
+
+  docker-release:
+    name: Publish tested multiarch image
+    if: >-
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'push' &&
+      needs.python-release.outputs.release_ready == 'true'
+    needs: python-release
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout the tested commit
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 1
+          ref: ${{ env.RELEASE_SHA }}
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+
+      - name: Resolve release and Docker binaries through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
+          for name in uv git jq docker; do
+            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
+            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
+          done
+          {
+            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
+            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
+            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
+            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
+          } >> "$GITHUB_ENV"
+
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+        with:
+          install: true
+
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      - name: Download exact tested image digests
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.CI_RUN_ID }}
+
+      - name: Verify digest set and collect tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          [[ "${#DIGESTS[@]}" -eq 2 ]]
+          for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
+
+          VERSION='${{ needs.python-release.outputs.version }}'
+          SHORT_SHA="${RELEASE_SHA::12}"
+          TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
+          TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
+          if [[ -z "$TAG_TARGET" ]]; then
+            TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
+            TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
+          fi
+          {
+            echo 'dockerhub_tags<<EOF'
+            if [[ "$RELEASE_BRANCH" == dev ]]; then
+              echo "${DOCKERHUB_IMAGE}:dev"
+            elif [[ "$RELEASE_BRANCH" == main ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            if [[ "$RELEASE_BRANCH" == dev ]]; then
+              echo "${GHCR_IMAGE}:dev"
+            elif [[ "$RELEASE_BRANCH" == main ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          TAG_ARGS=()
+          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
+          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          TAG_ARGS=()
+          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
+          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Verify published Docker images run
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION='${{ needs.python-release.outputs.version }}'
+          for image in "${DOCKERHUB_IMAGE}:${VERSION}" "${GHCR_IMAGE}:${VERSION}"; do
+            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
+            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
+            test "$INSTALLED_VERSION" = "$VERSION"
+          done
+
+      - name: Update Docker Hub README
+        uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox
+
+  cascade:
+    name: Cascade verified release
+    needs:
+      - python-release
+      - docker-release
+    permissions:
+      contents: read
+    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
+    with:
+      upstream_package: archivebox
+      upstream_version: ${{ needs.python-release.outputs.version }}
+    secrets:
+      release_token: ${{ secrets.RELEASE_GH_TOKEN }}

.github/workflows/test-parallel.yml

@@ -1,296 +1,154 @@
 name: Parallel Tests
 
 on:
-  workflow_dispatch:
-  pull_request:
-    branches: [dev, main, master]
-  push:
-    branches: [dev]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
+  CI_PYTHON_VERSION: "3.13"
 
 jobs:
   discover-tests:
-    name: Discover test files
+    name: Discover every test
     runs-on: ubuntu-24.04
     outputs:
-      test-files: ${{ steps.set-matrix.outputs.test-files }}
-      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
+      tests: ${{ steps.discover.outputs.tests }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - name: Discover test files
-        id: set-matrix
-        run: |
-          # Find all main test files
-          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
-
-          # Create JSON array with test file info
-          json_array="["
-          first=true
-          for test_file in $all_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            # Extract a display name for the test
-            name="main/$(basename $test_file .py | sed 's/^test_//')"
-
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\"}"
-          done
-          json_array+="]"
-
-          echo "test-files=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $all_tests | wc -w) test files"
-          echo "$json_array" | jq '.'
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Discover plugin tests
-        id: set-plugin-matrix
-        run: |
-          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
-
-          json_array="["
-          first=true
-          for plugin_name in $plugin_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            needs_chromium=false
-            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
-              needs_chromium=true
-            fi
-
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
-          done
-          json_array+="]"
-
-          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
-          echo "$json_array" | jq '.'
-
-  run-tests:
-    name: ${{ matrix.test.name }}
-    runs-on: ubuntu-24.04
-    needs: discover-tests
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
-        python: ["3.13"]
-
-    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
-          submodules: true
-          fetch-depth: 1
+          python-version: ${{ env.CI_PYTHON_VERSION }}
 
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v5
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
-          python-version: ${{ matrix.python }}
-          architecture: x64
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
-
-      - name: Run test - ${{ matrix.test.name }}
-        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
-
-  plugin-tests:
-    name: ${{ matrix.plugin.name }}
-    runs-on: ubuntu-24.04
-    needs: discover-tests
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      - name: Set up Python 3.13
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-3.13-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
-
-      - name: Select Chromium for extension-capable CDP
-        if: ${{ matrix.plugin.needs_chromium }}
+      - name: Resolve discovery tools through abxpkg
         env:
-          MIN_CHROMIUM_MAJOR: "149"
-          LIB_DIR: ${{ runner.temp }}/abx-lib
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
         run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          test -n "$ABXPKG_VERSION"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          export PYTHON_BINARY="$pythonLocation/bin/python"
+          test -x "$PYTHON_BINARY"
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
+          {
+            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
+            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
+          } >> "$GITHUB_ENV"
+          test -L "$ABXPKG_LIB_DIR/env/bin/python"
+          test -x "$ABXPKG_LIB_DIR/env/bin/python"
+          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
+          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
 
-          candidate="/usr/bin/chromium"
-          version=""
-          major=0
-          if [ -x "$candidate" ]; then
-            version="$("$candidate" --version || true)"
-            major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-            case "$major" in
-              ''|*[!0-9]*) major=0 ;;
-            esac
-          fi
-
-          if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Using runner Chromium: $candidate ($version)"
-            echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-            exit 0
-          fi
-
-          echo "Runner Chromium is unavailable or too old: ${version:-not found}"
-          echo "Installing Playwright Chromium into $LIB_DIR..."
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=npm \
-            --lib="$LIB_DIR" \
-            --postinstall-scripts=false \
-            --min-release-age=0 \
-            --no-cache \
-            --overrides='{"npm":{"install_args":["playwright@next"]}}' \
-            playwright
-          uv run --no-sync --no-sources abxpkg install \
-            --binproviders=playwright \
-            --lib="$LIB_DIR" \
-            --bin-dir="$LIB_DIR/env/bin" \
-            --min-version="$MIN_CHROMIUM_MAJOR" \
-            chromium
-
-          candidate="$LIB_DIR/env/bin/chromium"
-          if [ ! -x "$candidate" ]; then
-            echo "Playwright Chromium shim was not created at $candidate" >&2
-            exit 1
-          fi
-          version="$("$candidate" --version || true)"
-          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-          case "$major" in
-            ''|*[!0-9]*) major=0 ;;
-          esac
-          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "Playwright Chromium is still too old: ${version:-unknown}" >&2
-            exit 1
-          fi
-          echo "Using Playwright Chromium: $candidate ($version)"
-          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
-
-      - name: Run plugin tests - ${{ matrix.plugin.name }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+      - name: Discover every test exactly once
+        id: discover
         run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage
+          set -Eeuo pipefail
+          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
+          test "${#output[@]}" -eq 2
+          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
+          echo "${output[0]}"
+          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
+
+  tests:
+    name: ${{ matrix.test.name }}
+    needs: discover-tests
+    runs-on: ubuntu-24.04
+    env:
+      ABXPKG_LIB_DIR: /tmp/archivebox-lib
+      CHROME_HEADLESS: "true"
+      DATA_DIR: /tmp/archivebox-test-data
+      PERSONAS_DIR: /tmp/abx-personas
+      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
+
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          submodules: true
+          fetch-depth: 1
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: ${{ env.CI_PYTHON_VERSION }}
+          architecture: x64
+
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Resolve optional build dependencies through abxpkg
+        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
+        env:
+          TEST_EXTRA: ${{ matrix.test.extra }}
+          TEST_PATHS: ${{ matrix.test.paths_arg }}
+        run: |
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          test -n "$ABXPKG_VERSION"
+          dependency_sources=()
+          if [[ -n "$TEST_EXTRA" ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
+          fi
+          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
+          fi
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+            --install \
+            --no-cache \
+            --json \
+            --lib="$ABXPKG_LIB_DIR" \
+            "${dependency_sources[@]}"
+
+      - name: Install test dependencies
+        env:
+          TEST_EXTRA: ${{ matrix.test.extra }}
+        run: |
+          set -Eeuo pipefail
+          extra_args=()
+          if [[ -n "$TEST_EXTRA" ]]; then
+            extra_args=(--extra "$TEST_EXTRA")
+          fi
+          uv sync --no-cache --locked --dev "${extra_args[@]}"
+
+      - name: Run ${{ matrix.test.name }}
+        env:
+          TEST_PATHS: ${{ matrix.test.paths_arg }}
+        run: |
+          set -Eeuo pipefail
+          read -r -a test_paths <<< "$TEST_PATHS"
+          for test_path in "${test_paths[@]}"; do
+            test -f "$test_path"
+          done
+          for test_path in "${test_paths[@]}"; do
+            test_name="$(basename "$test_path" .py)"
+            test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
+            export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
+            export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
+            export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
+            mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
+            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
+              --basetemp="$RUNNER_TEMP/pytest/$test_slug"
+          done

.github/workflows/test.yml

@@ -1,156 +1,63 @@
-name: Run tests
-on: [push]
+name: Install and CLI Compatibility
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+on:
+  workflow_call:
 
 env:
   DOCKER_IMAGE: archivebox-ci
+  ARCHIVEBOX_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
 
 jobs:
-  python_tests:
+  install_cli_platform:
+    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
     runs-on: ${{ matrix.os }}
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
-        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: ["3.13"]
+        include:
+          - os: ubuntu-24.04
+            os_name: linux
+            python: "3.14"
+          - os: macos-15
+            os_name: macOS
+            python: "3.13"
+          - os: macos-15
+            os_name: macOS
+            python: "3.14"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      ### Setup Python & JS Languages
+      ### Set up Python
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: ${{ matrix.python }}
-          architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
         with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      ### Install Python & JS Dependencies
-      - name: Cache uv
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
 
       - name: Install dependencies with uv
+        run: uv sync --no-cache --locked --dev
+
+      - name: ArchiveBox full install check
         run: |
-          uv sync --dev --all-extras --no-sources --no-cache
-          uv pip install -e ./abxpkg -e ./abx-plugins[dev] -e ./abx-dl
-
-      ### Run the tests
-      - name: Directory listing for debugging
-        run: |
-          pwd
-          ls
-
-      - name: Archivebox version
-        run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
-
-      - name: Test built package with pytest
-        # TODO: remove this exception for windows once we get tests passing on that platform
-        if: ${{ !contains(matrix.os, 'windows') }}
-        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
-
-      - name: Run plugin tests
-        if: ${{ !contains(matrix.os, 'windows') }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
-
-  docker_tests:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
-      # - uses: satackey/action-docker-layer-caching@v0.0.8
-
-      - name: Build image
-        run: |
-          docker build . -t "$DOCKER_IMAGE"
-
-      - name: Init data dir
-        run: |
-          mkdir "${{ github.workspace }}/data"
-          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
-
-      - name: Run test server
-        run: |
-          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-          docker run --name www-nginx -p 80:80 -d nginx
-
-      - name: Add link
-        run: |
-          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
-
-      - name: Add stdin link
-        run: |
-          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
-
-      - name: List links
-        run: |
-          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
-          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
-
-      - name: Start docker-compose stack
-        run: |
-          docker-compose run archivebox init
-          docker-compose up -d
-          sleep 5
-          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
-          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
-
-      - name: Check added urls show up in index
-        run: |
-          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
-          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
-          docker-compose down || true
+          set -Eeuo pipefail
+          DATA_DIR="$PWD/tests/out/data"
+          mkdir -p "$DATA_DIR"
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status

.gitignore

@@ -8,6 +8,8 @@ __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/
+/docs/screenshots/
+/publicsite/screenshots/
 
 # Coverage
 .coverage

AGENTS.md

@@ -0,0 +1,80 @@
+# ArchiveBox Agent Guide
+
+ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
+
+## Shared Standards
+
+- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
+- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
+- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
+- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
+- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
+- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
+- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
+- Start behavior fixes with a red failing test when a test is requested or practical.
+- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
+- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
+
+## Development Setup
+
+```bash
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
+```
+
+Run collection commands from inside an initialized data directory:
+
+```bash
+cd data
+uv run --project .. archivebox status
+uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
+uv run --project .. archivebox run
+```
+
+## User-Facing Setup
+
+Recommended CLI install:
+
+```bash
+uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add --plugins=parse_txt_urls 'https://example.com/'
+```
+
+Alternative install methods:
+
+- Docker Compose / Docker
+- Homebrew
+- Debian package
+- pip
+
+## Basic Usage
+
+```bash
+cd ~/archivebox/data
+archivebox version
+archivebox help
+archivebox status
+archivebox install
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
+archivebox run
+```
+
+## Verification
+
+Use targeted tests for focused work:
+
+```bash
+uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
+uv run prek run --all-files
+```
+
+Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

CLAUDE.md

@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
-- Python 3.11+ (3.13 recommended)
-- uv package manager
-- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
-- Group related functions with common prefixes
-- Use `_` prefix for internal/private helpers within the same family
-- ALL logging-related methods MUST start with `log_` or `_log`
-- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
-- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
-- Create real SQLite databases with version-specific schemas
-- Seed with realistic test data
-- Run actual `python -m archivebox` commands via subprocess
-- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
-- `init` command must return exit code 0 (not `[0, 1]`)
-- Verify ALL data is preserved, not just "at least one"
-- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
-- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
-- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
-- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
-- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
-- Squashed migrations for clean installs
-- Individual migrations recorded for upgrades from dev branch
-- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
-- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
-- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
-- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
-- `dns` - DNS resolution info
-- `ssl` - SSL certificate info
-- `headers` - HTTP response headers
-- `redirects` - Capture redirect chains
-- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
-- `responses` - Capture network responses
-- `consolelog` - Capture console.log output
-- `title` - Extract page title
-- `accessibility` - Extract accessibility tree
-- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
-- `ublock` - uBlock Origin ad blocking
-- `istilldontcareaboutcookies` - Cookie banner dismissal
-- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
-- `modalcloser` - Modal dialog dismissal
-- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
-- `dom` - DOM snapshot extraction
-- `pdf` - Generate PDF snapshots
-- `screenshot` - Generate screenshots
-- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
-- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
-- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
-- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
-- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
-- Crawl groups multiple Snapshots from a single `add` command
-- Each `add` creates one Crawl with one or more Snapshots
-- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
-- Unit tests (pytest)
-- Integration tests
-- Dev server (manual testing)
-- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
-- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
-- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
-- **Reset**: Use `coverage erase` to start fresh for isolated measurements
-- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
-- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```

Dockerfile

@@ -1,38 +1,196 @@
-# This is the Dockerfile for ArchiveBox, it bundles the following main dependencies:
-#     python3.13, uv, python3-ldap
-#     curl, wget, git, dig, ping, tree, nano
-#     node, npm
-#     ArchiveBox and plugin runtime dependencies installed by archivebox init --install
-# Usage:
-#     git clone https://github.com/ArchiveBox/ArchiveBox && cd ArchiveBox
-#     docker build . -t archivebox
-#     docker run -v "$PWD/data":/data archivebox init
-#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
-#     docker buildx build . --platform=linux/amd64,linux/arm64 --push -t archivebox/archivebox:dev -t archivebox/archivebox:sha-abc123
-# Read more here: https://github.com/ArchiveBox/ArchiveBox#archivebox-development
+# syntax=docker/dockerfile:1.7
 
+# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
+# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
+# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
+# Build abx-dl first, then point this file at it:
+#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
+#       --build-context abxbus=../abxbus \
+#       --build-context abxpkg=../abxpkg \
+#       --build-context abx-plugins=../abx-plugins \
+#       -t archivebox/abx-dl:dev
+#   docker buildx build . -f Dockerfile \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.158 \
+#       -t archivebox:multistage
 
-#########################################################################################
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.158
 
-### Example: Using ArchiveBox in your own project's Dockerfile ########
-
-# FROM python:3.13-slim
-# WORKDIR /data
-# RUN pip install archivebox>=0.9.0   # use latest release here
-# RUN archivebox install
-# RUN useradd -ms /bin/bash archivebox && chown -R archivebox /data
-
-#########################################################################################
+FROM archivebox/sonic:1.4.9 AS sonic
+FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
 
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-ARG TARGETVARIANT=
+ARG TARGETVARIANT
+ARG ARCHIVEBOX_COMMIT_HASH=""
 
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04
+ENV TZ=UTC \
+    LANGUAGE=en_US:en \
+    LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
+    DEBIAN_FRONTEND=noninteractive \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    PYTHONIOENCODING=UTF-8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PIP_NO_COMPILE=1 \
+    PIP_ONLY_BINARY=aiohttp \
+    npm_config_loglevel=error
+
+ENV PYTHON_VERSION=3.13 \
+    NODE_VERSION=24
+
+ENV ARCHIVEBOX_USER=archivebox \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
+    IN_DOCKER=True
+
+ENV CODE_DIR=/app \
+    DATA_DIR=/data \
+    CONFIG_DIR=/opt/archivebox \
+    ABXPKG_LIB_DIR=/opt/archivebox/lib \
+    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
+    PERSONAS_DIR=/data/personas \
+    CHROME_HEADLESS=true \
+    CHROME_SANDBOX=false \
+    CHROME_ISOLATION=crawl \
+    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
+
+ENV TMP_DIR=/tmp/archivebox \
+    PIP_VENV_PYTHON=/venv/bin/python3 \
+    GOOGLE_API_KEY=no \
+    GOOGLE_DEFAULT_CLIENT_ID=no \
+    GOOGLE_DEFAULT_CLIENT_SECRET=no
+
+ENV HOME=/home/archivebox \
+    XDG_CONFIG_HOME=/home/archivebox/.config \
+    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
+    TIMEOUT=600
+
+ENV UV_COMPILE_BYTECODE=false \
+    UV_PYTHON_PREFERENCE=managed \
+    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/venv \
+    VIRTUAL_ENV=/venv \
+    PATH="/venv/bin:/opt/node/bin:$PATH"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
+WORKDIR "$CODE_DIR"
+
+RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
+    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
+    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
+    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
+    && uname -a \
+    && sed -n '1,7p' /etc/os-release \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env node \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env uv \
+    ) | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base AS archivebox-builder
+
+WORKDIR "$CODE_DIR"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
+    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
+    <<'EOF'
+echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
+echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
+echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
+echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+rm -f /etc/apt/apt.conf.d/docker-clean
+apt-get update -qq
+apt-get install -qq -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+/usr/bin/uv venv --no-cache --clear /venv --python "${PYTHON_VERSION}"
+/usr/bin/uv pip install --no-cache setuptools pip wheel
+
+mkdir -p /tmp/archivebox-uv-project
+/venv/bin/python - <<'PY'
+from pathlib import Path
+import json
+import re
+import urllib.request
+
+source = Path("/app/pyproject.toml")
+target = Path("/tmp/archivebox-uv-project/pyproject.toml")
+text = source.read_text()
+text = text.replace(
+    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+    'environments = ["sys_platform == \'linux\'"]',
+)
+
+# Docker builds need the just-published internal abx wheels immediately, but
+# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
+# Docker-only dependency view from the version JSON so the published package
+# metadata stays normal while image builds remain resumable after a release.
+for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
+    match = re.search(
+        rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
+        text,
+    )
+    if not match:
+        continue
+    version = match.group("version")
+    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+        data = json.load(response)
+    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+    text = re.sub(
+        rf'"{re.escape(package)}(?:==|>=)[^"]+"',
+        f'"{package} @ {wheel_url}"',
+        text,
+        count=1,
+    )
+
+target.write_text(text)
+PY
+
+/usr/bin/uv sync \
+    --project /tmp/archivebox-uv-project \
+    --no-cache \
+    --no-dev \
+    --inexact \
+    --no-install-project \
+    --no-install-workspace \
+    --no-sources
+ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$ABXPKG_LIB_DIR" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
+/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
+while IFS= read -r -d '' native_library; do
+    magic=''
+    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
+        "$ABXPKG_LIB_DIR/env/bin/strip" --strip-unneeded "$native_library" || exit $?
+    fi
+done < /tmp/archivebox-native-libraries
+rm -f /tmp/archivebox-native-libraries
+rm -f /venv/bin/uv /venv/bin/uvx
+abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get autoremove -y
+/usr/bin/find "$ABXPKG_LIB_DIR/env/bin" -maxdepth 1 -type l -name strip -delete
+rm -rf /var/lib/apt/lists/*
+EOF
+
+COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
+RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+    && COMMIT_HASH="$( \
+        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+            echo "$ARCHIVEBOX_COMMIT_HASH"; \
+        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
+            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+                echo "$HEAD_REF"; \
+            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
+                REF_PATH="${HEAD_REF#ref: }"; \
+                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
+            fi; \
+        fi)" \
+    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
+    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && find /venv -exec touch -h -d "@$(date +%s)" {} + \
+    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
+
+FROM archivebox-runtime-base
 
 LABEL name="archivebox" \
     maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@@ -43,357 +201,65 @@ LABEL name="archivebox" \
     org.opencontainers.image.vendor="ArchiveBox" \
     org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
     org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile" \
-    # TODO: release ArchiveBox as a Docker Desktop extension (requires these labels):
-    # https://docs.docker.com/desktop/extensions-sdk/architecture/metadata/
-    com.docker.desktop.extension.api.version=">= 1.4.7" \
-    com.docker.desktop.extension.icon="https://archivebox.io/icon.png" \
-    com.docker.extension.publisher-url="https://archivebox.io" \
-    com.docker.extension.screenshots='[{"alt": "Screenshot of Admin UI", "url": "https://github.com/ArchiveBox/ArchiveBox/assets/511499/e8e0b6f8-8fdf-4b7f-8124-c10d8699bdb2"}]' \
-    com.docker.extension.detailed-description='See here for detailed documentation: https://wiki.archivebox.io' \
-    com.docker.extension.changelog='See here for release notes: https://github.com/ArchiveBox/ArchiveBox/releases' \
-    com.docker.extension.categories='database,utility-tools'
+    com.docker.image.source.entrypoint="Dockerfile"
 
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-######### Environment Variables #################################
-
-# Global build-time and runtime environment constants + default pkg manager config
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-# Language Version config
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-# Non-root User config
-ENV ARCHIVEBOX_USER="archivebox" \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-# ArchiveBox Source Code + Lib + Data paths
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    SONIC_BINARY=/opt/archivebox/lib/env/bin/sonic \
-    PLAYWRIGHT_BROWSERS_PATH=/browsers
-
-# Bash SHELL config
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"] 
-
-######### System Environment ####################################
-
-# Detect ArchiveBox version number by reading pyproject.toml (also serves to invalidate the entire build cache whenever pyproject.toml changes)
-WORKDIR "$CODE_DIR"
-
-# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up back-to-back Docker builds)
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-# Print debug info about build and save it to disk, for human eyes only, not used by anything else
-RUN (echo "[i] Docker build for ArchiveBox starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && echo \
-    && echo "PYTHON=${PYTHON_VERSION} NODE=${NODE_VERSION} PATH=${PATH}" \
-    && echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR}" \
-    && echo \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which bash && bash --version | sed -n '1p' \
-    && which dpkg && dpkg --version | sed -n '1p' \
-    && echo -e '\n\n' && env && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-# Create non-privileged user for archivebox and chrome
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system $ARCHIVEBOX_USER \
-    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u $ARCHIVEBOX_USER) PGID=$(id -g $ARCHIVEBOX_USER)\n\n" \
-    | tee -a /VERSION.txt
-    # DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
-    # https://docs.linuxserver.io/general/understanding-puid-and-pgid
-
-# Install system apt dependencies (adding backports to access more recent apt updates)
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing base system dependencies for $TARGETPLATFORM..." \
-    && mkdir -p /etc/apt/keyrings \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        # 1. packaging dependencies
-        apt-transport-https ca-certificates apt-utils gnupg2 curl wget \
-        # 2. docker and init system dependencies
-        zlib1g-dev dumb-init gosu cron unzip grep dnsutils git ripgrep python3.12-venv default-jre-headless \
-        # 3. frivolous CLI helpers to make debugging failed archiving easier
-        tree nano iputils-ping \
-        # nano iputils-ping dnsutils htop procps jq yq
-    && rm -rf /var/lib/apt/lists/*
-
-# Install sonic search backend
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-RUN (which sonic && sonic --version) | tee -a /VERSION.txt
 
-######### Language Environments ####################################
+COPY --from=archivebox-builder /venv /venv
+COPY --from=archivebox-builder /app /app
+COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
 
-# Set up Python environment
-# NOT NEEDED because we're using a pre-built python image, keeping this here in case we switch back to custom-building our own:
-#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-#    --mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-# RUN echo "[+] APT Installing PYTHON $PYTHON_VERSION for $TARGETPLATFORM (skipped, provided by base image)..." \
-    # && apt-get update -qq \
-    # && apt-get install -qq -y --no-upgrade \
-    #     python${PYTHON_VERSION} python${PYTHON_VERSION}-minimal python3-pip python${PYTHON_VERSION}-venv pipx \
-    # && rm -rf /var/lib/apt/lists/* \
-    # tell PDM to allow using global system python site packages
-    # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
-    # && ln -s "$(which python${PYTHON_VERSION})" /usr/bin/python \
-    # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
-    # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
-    # && python3 -m venv --system-site-packages $GLOBAL_VENV \
-    # && python3 -m venv $GLOBAL_VENV \
-    # install global dependencies / python build dependencies in GLOBAL_VENV
-    # && pip install --upgrade pip setuptools wheel \
-    # Save version info
-    # && ( \
-    #     which python3 && python3 --version | grep " $PYTHON_VERSION" \
-    #     && which pip && pip --version \
-    #     # && which pdm && pdm --version \
-    #     && echo -e '\n\n' \
-    # ) | tee -a /VERSION.txt
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
+    && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
+    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
+    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
+    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
+    && openssl rand -hex 16 > /etc/machine-id \
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
-
-# Set up Node environment from the official platform tarball. This avoids
-# NodeSource apt dependencies pulling Ubuntu's python3-minimal postinst into
-# emulated Docker builds.
-RUN --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    case "$TARGETARCH" in \
-        amd64) NODE_DIST_ARCH="x64" ;; \
-        arm64) NODE_DIST_ARCH="arm64" ;; \
-        *) echo "Unsupported TARGETARCH=$TARGETARCH for Node binary install" >&2; exit 1 ;; \
-    esac \
-    && NODE_TARBALL="node-v${NODE_VERSION}-linux-${NODE_DIST_ARCH}.tar.gz" \
-    && echo "[+] Installing NODE $NODE_VERSION for linux/${NODE_DIST_ARCH}..." \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" \
-    && curl -fsSLO "https://nodejs.org/dist/v${NODE_VERSION}/SHASUMS256.txt" \
-    && grep "  ${NODE_TARBALL}$" SHASUMS256.txt | sha256sum -c - \
-    && tar -xzf "$NODE_TARBALL" -C /usr/local --strip-components=1 --no-same-owner \
-    && rm "$NODE_TARBALL" SHASUMS256.txt \
-    # Save version info
-    && ( \
-        which node && node --version \
-        && which npm && npm --version \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-# Set up uv and main app /venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/bin sh
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv
-WORKDIR "$CODE_DIR"
-# COPY --chown=root:root --chmod=755 pyproject.toml "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] UV Creating /venv using python ${PYTHON_VERSION} for ${TARGETPLATFORM}..." \
-    && uv venv /venv --python ${PYTHON_VERSION}
-ENV VIRTUAL_ENV=/venv PATH="/venv/bin:$PATH"
-RUN uv pip install setuptools pip \
-    && ( \
-        which python3 && python3 --version \
-        && which uv && uv self version \
-        && uv python find --system && uv python find \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt
-
-
-######### ArchiveBox & Extractor Dependencies ##################################
-
-# Install ArchiveBox C-compiled/apt-installed Python dependencies in app /venv (currently only used for python-ldap)
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    #--mount=type=cache,target=/root/.cache/pip,sharing=locked,id=pip-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing + Compiling python3-ldap for PIP archivebox[ldap] on ${TARGETPLATFORM}..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc \
-        python3-dev libssl-dev libldap2-dev libsasl2-dev python3-ldap \
-        python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps \
-    && uv pip install \
-        "python-ldap>=3.4.3" \
-    && apt-get purge -y \
-        python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-
-# Runtime config used by plugin hooks. Plugin binaries and npm packages are
-# installed into LIB_DIR below by archivebox init --install and resolved from
-# LIB_DIR by ArchiveBox/abxpkg, not by mutating the container PATH.
-ENV PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-######### Build Dependencies ####################################
-
-
-# Install ArchiveBox Python venv dependencies from pyproject.toml.
-RUN --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[+] PIP Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential gcc python3-dev \
-    && uv --no-cache sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y python3-dev build-essential gcc \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-    # installs the pip packages that archivebox depends on, defined in pyproject.toml dependencies
-
-# Setup ArchiveBox runtime config
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/usr/bin/python3.12 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-WORKDIR "$DATA_DIR"
-RUN openssl rand -hex 16 > /etc/machine-id \
-    && mkdir -p "$DATA_DIR" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" \
-    && mkdir -p "$TMP_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$TMP_DIR" \
-    && mkdir -p "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && echo -e "\nTMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-# Pre-bake plugin-managed runtime dependencies using the same abx-dl installer
-# path users run later, before copying ArchiveBox source so source-only edits do
-# not invalidate the heavy browser/plugin dependency layer.
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.npm,sharing=locked,id=npm-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/puppeteer,sharing=locked,id=puppeteer-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/ms-playwright,sharing=locked,id=browsers-$TARGETARCH$TARGETVARIANT \
-    echo "[+] Installing plugin runtime dependencies into $LIB_DIR..." \
-    && export PERSONAS_DIR="$LIB_DIR/personas" \
-    && export CHROME_USER_DATA_DIR="$LIB_DIR/chrome_profile" \
-    && mkdir -p "$LIB_DIR" \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
-    && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \
-    && if [ "$TARGETARCH" = "arm64" ]; then \
-        abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
-        abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
-    fi \
-    && TIMEOUT=600 PUID=0 PGID=0 abx-dl plugins --install \
-    && abxpkg install --no-cache --binproviders=chromewebstore --overrides='{"chromewebstore":{"install_args":["fpeoodllldobpkbkabpblcfaogecpndd","--name=archivewebpage"]}}' archivewebpage \
-    && test -f "$LIB_DIR/chromewebstore/extensions/fpeoodllldobpkbkabpblcfaogecpndd__archivewebpage/manifest.json" \
-    && mkdir -p "$LIB_DIR/env/bin" \
-    && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
-    && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
-    && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
-    && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
-    && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
-    && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$LIB_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf "$LIB_DIR/personas" "$LIB_DIR/chrome_profile" /opt/archivebox/lib-layer \
-    && mkdir -p /opt/archivebox/lib-layer \
-    && cp -a "$LIB_DIR"/. /opt/archivebox/lib-layer/ \
-    && apt-get purge -y build-essential \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" /opt/archivebox/lib-layer
-
-RUN rm -rf "$LIB_DIR" \
-    && mv /opt/archivebox/lib-layer "$LIB_DIR" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR"
-
-# Install ArchiveBox Python package from the checked-out source.
-WORKDIR "$CODE_DIR"
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && pip install \
-        --no-deps \
-        "$CODE_DIR" \
-    && ( \
-        pip show archivebox \
-        && which archivebox \
-        && echo -e '\n\n' \
-    ) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-    # installs archivebox itself, and any other vendored packages in pkgs/*, defined in pyproject.toml workspaces
-
-# Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
+    && HOME="$TMP_DIR" archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+        2>/dev/null || true) \
+    && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
-# Print version for nice docker finish summary
-RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
+RUN chmod +x "$CODE_DIR"/bin/*.sh \
+    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+
+RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
+    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
+
+RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
+    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
     && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
     ) | tee -a /VERSION.txt
 
-# Verify ArchiveBox is installed and write full version/dependency info.
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage gallerydl liteparse mercury papersdl readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-####################################################
-
-# Expose ArchiveBox's main interfaces to the outside world
 WORKDIR "$DATA_DIR"
 VOLUME "$DATA_DIR"
 EXPOSE 8000

Dockerfile.multistage

@@ -1,227 +0,0 @@
-# syntax=docker/dockerfile:1.7
-
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile.multistage \
-#       --build-context abx-dl=docker-image://archivebox/abx-dl:dev \
-#       --build-arg ABX_DL_IMAGE=abx-dl \
-#       -t archivebox:multistage
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT=
-ARG ABX_DL_IMAGE=abx-dl
-
-FROM ${ABX_DL_IMAGE} AS abx-dl
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ubuntu:24.04 AS archivebox-runtime-base
-
-ARG TARGETPLATFORM=linux/amd64
-ARG TARGETOS=linux
-ARG TARGETARCH=amd64
-ARG TARGETVARIANT
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV UV_COMPILE_BYTECODE=0 \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox base runtime dependencies for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y \
-        apt-transport-https apt-utils ca-certificates curl wget gnupg2 \
-        dumb-init gosu unzip git grep ripgrep dnsutils iputils-ping procps tree nano \
-        cron openssl xz-utils zlib1g libldap2 libsasl2-2 libssl3 libsqlite3-0 \
-        libasound2t64 libatk-bridge2.0-0 libatk1.0-0 libcairo2 libcups2 \
-        libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
-        libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxdamage1 \
-        libxext6 libxfixes3 libxkbcommon0 libxrandr2 libxshmfence1 \
-        fonts-liberation fonts-noto-color-emoji xdg-utils \
-        ffmpeg imagemagick tesseract-ocr tesseract-ocr-eng openjdk-21-jre-headless \
-    && rm -rf /var/lib/apt/lists/*
-
-# Runtime-owned layers copied from the abx-dl image.
-COPY --from=abx-dl /bin/uv /bin/uv
-COPY --from=abx-dl /opt/uv/python /opt/uv/python
-COPY --from=abx-dl /opt/node /opt/node
-COPY --from=abx-dl /venv /venv
-COPY --from=abx-dl /VERSION.txt /ABX-DL-VERSION.txt
-
-RUN (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which python3 && python3 --version \
-    && which node && node --version \
-    && which uv && uv self version \
-    ) | tee -a /VERSION.txt
-
-ENV PYTHONDONTWRITEBYTECODE=1
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && find /venv -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && uv pip install --no-deps "$CODE_DIR" \
-    && (uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete
-
-FROM archivebox-runtime-base
-
-LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
-    homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile.multistage"
-
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-
-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-COPY --from=abx-dl --chown=911:911 /opt/archivebox/lib /opt/archivebox/lib
-
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && groupadd --system "$ARCHIVEBOX_USER" \
-    && useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && mkdir -p "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && PUID=0 PGID=0 archivebox init \
-    && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        2>/dev/null || true)
-
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install gallerydl liteparse mercury papersdl readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
-    && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && ! command -v gcc \
-    && ! command -v g++ \
-    && ! command -v make \
-    && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
-    && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-VOLUME "$DATA_DIR"
-EXPOSE 8000
-
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
-
-ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -8,14 +8,8 @@
 
 <br/>
 
-<!--<a href="http://webchat.freenode.net?channels=ArchiveBox&uio=d4"><img src="https://img.shields.io/badge/Community_chat-IRC-%2328A745.svg"/></a>-->
-
 <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> &nbsp; <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a> &nbsp; <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls"/></a> <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae"/></a> <a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8"/></a>
 
-<!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
-curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instructions below)
-</code></pre>-->
-
 </div>
 <hr/>
 <br/>
@@ -29,7 +23,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `uv tool install --python 3.13 --prerelease explicit --upgrade 'archivebox>=0.9.0rc0,<0.10'` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on Linux/macOS.
 
 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
@@ -52,7 +46,7 @@ It also detects any content featured *inside* pages & extracts it out into a fol
 - 💾 **Github**/**Gitlab**/etc. links ➡️ `clone of GIT source code`, `README`, `images`, ...
 - ✨ *and more, see [Output Formats](#output-formats) below...*
 
-You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `pip`-installed CLI, Python API, and SQLite3 APIs. 
+You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `uv`-installed CLI, Python API, and SQLite3 APIs.
 All the ways of using it are equivalent, and provide matching features like adding tags, scheduling regular crawls, viewing logs, and more...
 
 <br/>
@@ -68,7 +62,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <br/>
 
 
-**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `pip` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
+**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `uv` / `brew` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
 
 
 <details>
@@ -76,23 +70,26 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <br/>
 <pre lang="bash"><code style="white-space: pre-line"># Option A: Get ArchiveBox with Docker Compose (recommended):
 mkdir -p ~/archivebox/data && cd ~/archivebox
-curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml   # edit options in this file as-needed
-docker compose run archivebox init
-# docker compose run archivebox add 'https://example.com'
-# docker compose run archivebox help
-# docker compose up
+curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
+docker compose pull
+docker compose up -d --wait                                                # initializes new collections automatically
+docker compose exec archivebox archivebox manage createsuperuser          # create the first Web UI user
+# docker compose run --rm archivebox add 'https://example.com'
+# docker compose run --rm archivebox help
 <br/>
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init
-# docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
-# docker run -it -v $PWD:/data archivebox/archivebox help
-# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
+docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
+docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev install
+docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev manage createsuperuser
+docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
+# docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
+# docker run -it -v $PWD:/data archivebox/archivebox:dev help
 <br/>
 <br/>
-# Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install archivebox
+# Option C: Or install it with uv (see Quickstart below for apt, brew, and more)
+uv tool install --python 3.13 --prerelease explicit --upgrade 'archivebox>=0.9.0rc0,<0.10'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -106,7 +103,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
 <sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sub>
 </details>
 <br/>
 
@@ -171,14 +168,14 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 
 # Quickstart
 
-**🖥&nbsp; [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker) &nbsp; **👾&nbsp; CPUs:** `amd64` (`x86_64`), `arm64`, `arm7` <sup>(raspi>=3)</sup><br/>
+**🖥&nbsp; [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Ubuntu, macOS, Docker &nbsp; **👾&nbsp; CPUs:** `amd64` (`x86_64`), `arm64`<br/>
 
 <br/>
 
 #### ✳️&nbsp; Easy Setup
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux/Windows) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker-compose</code></b>  (macOS/Linux) &nbsp; <b>👈&nbsp; recommended</b> &nbsp; <i>(click to expand)</i></summary>
 <br/>
 <i>👍 Docker Compose is recommended for the easiest install/update UX + best security + all <a href="#dependencies">extras</a> out-of-the-box.</i>
 <br/><br/>
@@ -186,18 +183,18 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Download the <a href="https://raw.githubusercontent.com/ArchiveBox/ArchiveBox/dev/docker-compose.yml" download><code>docker-compose.yml</code></a> file into a new empty directory (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox
-# Read and edit docker-compose.yml options as-needed after downloading
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
+docker compose pull
 </code></pre></li>
-<li>Run the initial setup to create an admin user (or set ADMIN_USER/PASS in docker-compose.yml)
-<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init
+<li>Start the server, which initializes a new collection automatically, then create the first admin user.
+<pre lang="bash"><code style="white-space: pre-line">docker compose up -d --wait
+docker compose exec archivebox archivebox manage createsuperuser
 </code></pre></li>
-<li>Next steps: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker compose up
-# completely optional, CLI can always be used without running a server
-# docker compose run [-T] archivebox [subcommand] [--help]
-docker compose run archivebox add 'https://example.com'
-docker compose run archivebox help
+<li>Next steps: Log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">
+# run CLI commands inside the server container started above
+docker compose exec archivebox archivebox add 'https://example.com'
+docker compose exec archivebox archivebox help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -208,20 +205,21 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker run</code></b>  (macOS/Linux/Windows)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447182-29758200-af0b-11eb-97bd-58723fee62ab.png" alt="Docker" height="28px" align="top"/> <code>docker run</code></b>  (macOS/Linux)</summary>
 <br/>
 <ol>
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init
+docker run --rm -v $PWD:/data -it archivebox/archivebox:dev init
+docker run --rm -v $PWD:/data -it archivebox/archivebox:dev install
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
+<li>Optional: Start the server then log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 # completely optional, CLI can always be used without running a server
-# docker run -v $PWD:/data -it [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox help
+# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -256,27 +254,26 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
+<li>Install <a href="https://docs.astral.sh/uv/getting-started/installation/">uv</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>uv</code>.
+<pre lang="bash"><code style="white-space: pre-line">uv tool install --python 3.13 --prerelease explicit --upgrade 'archivebox>=0.9.0rc0,<0.10'
 archivebox version
-# install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
-#    python@3.13 node curl wget git ripgrep ...
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Install: Bare Metal</a> Wiki for full install instructions for each OS...</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # for example
-archivebox init     # instantialize a new collection
+archivebox init     # initialize a new collection
 archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
+archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -286,7 +283,7 @@ archivebox help
 
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
 <br/>
-<sub>See the <a href="https://github.com/ArchiveBox/pip-archivebox"><code>pip-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>See the <a href="https://docs.astral.sh/uv/guides/tools/"><code>uv tool</code> documentation</a> for more details about this installation method.</sub>
 <br/><br/>
 </details>
 
@@ -295,23 +292,25 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><b><img src="https://user-images.githubusercontent.com/511499/117448075-49597580-af0c-11eb-91ba-f34fff10096b.png" alt="aptitude" height="28px" align="top"/> <code>apt</code></b> (Ubuntu/Debian/etc.)</summary>
 <br/>
 <ol>
-<li>Download and install the <code>.deb</code> package from the <a href="https://github.com/ArchiveBox/ArchiveBox/releases">latest release</a>.
-<pre lang="bash"><code style="white-space: pre-line"># download the .deb for your architecture (amd64 or arm64)
-ARCH="$(dpkg --print-architecture)"
-VERSION="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest | python3 -c "import sys,json; print(json.load(sys.stdin)['tag_name'].lstrip('v'))")"
-curl -fsSL "https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${VERSION}_${ARCH}.deb" -o /tmp/archivebox.deb
-sudo apt install /tmp/archivebox.deb
-archivebox version                         # make sure all dependencies are installed
+<li>Add the third-party ArchiveBox apt repo and install <code>archivebox</code>.
+<pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
+sudo apt update
+sudo apt install archivebox
+(cd /tmp && archivebox version)            # make sure the package is installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
-<pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+<pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox install
+archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
+archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -319,19 +318,21 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details, and run <code>sudo archivebox install</code> only if you want it to install missing system packages via apt.</sub>
 <br/><br/>
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS only)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
 <br/>
+Run Homebrew as your normal non-root user on both macOS and Linux; do not use <code>sudo brew</code>.
 <ol>
 <li>Install <a href="https://brew.sh/#install">Homebrew</a> on your system (if not already installed).</li>
 <li>Install the ArchiveBox package using <code>brew</code>.
 <pre lang="bash"><code style="white-space: pre-line">brew tap archivebox/archivebox
+brew trust archivebox/archivebox
 brew install archivebox
-archivebox version                         # make sure all dependencies are installed
+archivebox version                         # verify the installed version
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for more granular instructions for macOS... ➡️</i>
 </li>
@@ -341,8 +342,9 @@ archivebox init
 archivebox install
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
+archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -359,11 +361,11 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
 <br/>
 
-> *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
+> *Warning: These are contributed by external volunteers and may lag behind the official `uv` and Docker channels.*
 
 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>uv</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -399,7 +401,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <summary><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" height="28px" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
 <br/>
 
-> *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
+> *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
 
 <ul>
 <li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
@@ -484,11 +486,11 @@ cd ~/archivebox/data         # IMPORTANT: cd into the directory
 archivebox version
 archivebox help
 
-# equivalent: docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox help
+# equivalent: docker compose run --rm archivebox [subcommand] [--help]
+docker compose run --rm archivebox help
 
-# equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -it -v $PWD:/data archivebox/archivebox help
+# equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
 # optional: import your browser cookies into a persona for logged-in archiving
 archivebox persona create --import=chrome personal
@@ -509,13 +511,15 @@ archivebox persona create --import=chrome personal
 <summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
+# make sure you have uv-installed ArchiveBox and it's available in your $PATH first
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+archivebox init
+archivebox install
+archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
-archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -528,11 +532,12 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
-# docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
-docker compose run archivebox version
-docker compose run archivebox help
-docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+# docker compose run --rm archivebox [subcommand] [--help]
+docker compose run --rm archivebox init
+docker compose run --rm archivebox install
+docker compose run --rm archivebox version
+docker compose run --rm archivebox help
+docker compose run --rm archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -546,12 +551,13 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you create and cd into in a new empty directory first  
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
-docker run -v $PWD:/data -it archivebox/archivebox version
-docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -v $PWD:/data -it archivebox/archivebox:dev version
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
+docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
+# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -563,7 +569,7 @@ docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.y
 <pre lang="bash"><code style="white-space: pre-line">
 archivebox shell           # explore the Python library API in a REPL
 sqlite3 ./index.sqlite3    # run SQL queries directly on your index
-ls ./archive/*/index.html  # or inspect snapshot data directly on the filesystem
+find ./archive/users -path '*/snapshots/*/*/*/index.html'  # inspect snapshot data directly
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage">Python Shell</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage">SQL API</a>, and <a href="https://github.com/ArchiveBox/ArchiveBox#archive-layout">Disk Layout</a> wikis. ➡️</i>
 </details>
@@ -574,21 +580,21 @@ ls ./archive/*/index.html  # or inspect snapshot data directly on the filesystem
 <details>
 <summary><b>🖥&nbsp; Web UI & API Usage</b></summary>
 <pre lang="bash"><code style="white-space: pre-line">
-# Start the server on bare metal (pip/apt/brew/etc):
+# Start the server on bare metal (uv/apt/brew):
 archivebox manage createsuperuser              # create a new admin user via CLI
 archivebox server 0.0.0.0:8000                 # start the server
 <br/>
 # Or with Docker Compose:
-nano docker-compose.yml                        # setup initial ADMIN_USERNAME & ADMIN_PASSWORD
+docker compose run --rm archivebox manage createsuperuser
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuperuser
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+docker run -v $PWD:/data -it archivebox/archivebox:dev manage createsuperuser
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -596,10 +602,10 @@ docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 
 <pre lang="bash"><code style="white-space: pre-line">
 archivebox config --set PUBLIC_ADD_VIEW=True   # allow guests to submit URLs 
-archivebox config --set PUBLIC_SNAPSHOTS=True  # allow guests to see snapshot content
+archivebox config --set PERMISSIONS=public     # make newly added snapshots public
 archivebox config --set PUBLIC_INDEX=True      # allow guests to see list of all snapshots
 # or
-docker compose run archivebox config --set ...
+docker compose run --rm archivebox config --set PERMISSIONS=public
 
 # restart the server to apply any config changes
 </code></pre>
@@ -610,15 +616,15 @@ docker compose run archivebox config --set ...
 
 > [!TIP]
 > Whether in Docker or not, ArchiveBox commands work the same way, and can be used to access the same data on-disk.
-> For example, you could run the Web UI in Docker Compose, and run one-off commands with `pip`-installed ArchiveBox.
+> For example, you could run the Web UI in Docker Compose, and run one-off commands with `uv`-installed ArchiveBox.
 
 <details>
 <summary><i>Expand to show comparison...</i></summary><br/>
 
 <pre lang="bash"><code style="white-space: pre-line">
-archivebox add --depth=1 'https://example.com'                     # add a URL with pip-installed archivebox on the host
-docker compose run archivebox add --depth=1 'https://example.com'                       # or w/ Docker Compose
-docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
+archivebox add --depth=1 'https://example.com'                     # add a URL with uv-installed archivebox on the host
+docker compose run --rm archivebox add --depth=1 'https://example.com'                  # or w/ Docker Compose
+docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
 </code></pre>
 
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker</a> wiki. ➡️</i>
@@ -679,15 +685,15 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
 
 # if using Docker, add -i when piping stdin:
-# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox add
+# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
 # if using Docker Compose, add -T when piping stdin / stdout:
-# echo 'https://example.com' | docker compose run -T archivebox add
+# echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
@@ -745,10 +751,10 @@ ArchiveBox can be configured via environment variables, by using the `archivebox
 archivebox config --get CHROME_BINARY           # view a specific value
 <br/>
 archivebox config --set CHROME_BINARY=chromium  # persist a config using CLI
+# OR edit ArchiveBox.conf and add this under its existing [ARCHIVING_CONFIG] section:
+CHROME_BINARY=chromium
 # OR
-echo CHROME_BINARY=chromium >> ArchiveBox.conf  # persist a config using file
-# OR
-env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
+env CHROME_BINARY=chromium archivebox version   # run with a one-off config
 </code></pre>
 <sub>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sub>
 </details><br/>
@@ -760,13 +766,13 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 <summary><i>Expand to see the most common options to tweak...</i></summary>
 <pre lang="bash"><code style="white-space: pre-line">
 # e.g. archivebox config --set TIMEOUT=120
-# or   docker compose run archivebox config --set TIMEOUT=120
+# or   docker compose run --rm archivebox config --set TIMEOUT=120
 <br/>
 TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
-PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
+PERMISSIONS=public         # default: public visibility for newly added snapshots
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
 <br/>
 USER_AGENT="Mozilla/5.0 ..."  # change this to get around bot blocking
@@ -819,14 +825,10 @@ These optional subdependencies used for archiving sites include:
 <li>and more as we grow...</li>
 </ul>
 
-You don't need to install every dependency to use ArchiveBox. ArchiveBox will automatically disable extractors that rely on dependencies that aren't installed, based on what is configured and available in your <code>$PATH</code>.
-  
-If not using Docker, make sure to keep the dependencies up-to-date yourself and check that ArchiveBox isn't reporting any incompatibility with the versions you install.
+You don't need to install every dependency by hand. ArchiveBox resolves every extractor dependency through <code>abxpkg</code>: it uses a compatible host installation when one is already available, and otherwise installs and manages the dependency for you.
 
-<pre lang="bash"><code style="white-space: pre-line">#install python3 and archivebox with your system package manager
-# apt/brew/pip/etc install ... (see Quickstart instructions above)
+<pre lang="bash"><code style="white-space: pre-line"># install uv + archivebox first (see Quickstart instructions above)
 <br/>
-which -a archivebox    # see where you have installed archivebox
 archivebox install     # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
@@ -859,7 +861,7 @@ All <code>archivebox</code> CLI commands are designed to be run from inside an A
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # just an example, can be anywhere
 archivebox init</code></pre>
 
-The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard <code>index.sqlite3</code> database in the root of the data folder (it can also be <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">exported as static JSON/HTML</a>), and the archive snapshots are organized by date-added timestamp in the <code>data/archive/</code> subfolder.
+The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard <code>index.sqlite3</code> database in the root of the data folder (it can also be <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">exported as static JSON/HTML</a>). Snapshot data is organized by user, date, domain, and UUID under <code>data/archive/users/</code>.
 
 <img src="https://user-images.githubusercontent.com/511499/117453293-c7b91600-af12-11eb-8a3f-aa48b0f9da3c.png" width="400px" align="right" style="float: right"/>
 
@@ -868,18 +870,23 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
     index.sqlite3
     ArchiveBox.conf
     archive/
-        ...
-        1617687755/
-            index.html
-            index.json
-            screenshot.png
-            media/some_video.mp4
-            warc/1617687755.warc.gz
-            git/somerepo.git
-            ...
+        1617687755 -> users/admin/snapshots/20210406/example.com/SNAPSHOT_UUID/
+        users/
+            admin/
+                snapshots/
+                    20210406/
+                        example.com/
+                            SNAPSHOT_UUID/
+                                index.html
+                                index.jsonl
+                                screenshot/screenshot.png
+                                ytdlp/media/some_video.mp4
+                                wget/warc/example.com.warc.gz
+                                git/somerepo.git
+                                ...
 </code></pre>
 
-Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <code>index.json</code> and <code>index.html</code> describing its contents, and the snapshot extractor outputs are plain files within the folder.
+Each snapshot subfolder includes static metadata and plain extractor output files. ArchiveBox also maintains a backwards-compatible <code>data/archive/TIMESTAMP</code> symlink for each snapshot.
 
 <h4>Learn More</h4>
 <ul>
@@ -913,7 +920,7 @@ archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
 
 # (if using Docker Compose, add the -T flag when piping)
-# docker compose run -T archivebox list --html 'https://example.com' > index.json
+# docker compose run --rm -T archivebox list --html 'https://example.com' > index.html
 </code></pre>
 
 The paths in the static exports are relative, make sure to keep them next to your `./archive` folder when backing them up or viewing them.
@@ -923,7 +930,7 @@ The paths in the static exports are relative, make sure to keep them next to you
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">Wiki: Publishing Your Archive (Exporting as Static HTML)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#publishing">Wiki: Security Overview (Publishing)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#public_index--public_snapshots--public_add_view">Wiki: Configuration (<code>PUBLIC_INDEX</code>, <code>PUBLIC_SNAPSHOTS</code>, <code>PUBLIC_ADD_VIEW</code>)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions">Wiki: Configuration (<code>PERMISSIONS</code>, <code>PUBLIC_INDEX</code>, <code>PUBLIC_ADD_VIEW</code>)</a></li>
 </ul>
 
 </details>
@@ -954,7 +961,7 @@ archivebox add 'https://vimeo.com/somePrivateVideo'
 
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
-archivebox config --set PUBLIC_SNAPSHOTS=False
+archivebox config --set PERMISSIONS=private
 archivebox config --set PUBLIC_ADD_VIEW=False
 archivebox manage createsuperuser
 </code></pre>
@@ -970,8 +977,8 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir">Wiki: Configuration (<code>CHROME_USER_DATA_DIR</code>)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#cookies_file">Wiki: Configuration (<code>COOKIES_FILE</code>)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">Wiki: Personas (browser profiles and cookies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
 </details>
@@ -980,7 +987,7 @@ archivebox manage createsuperuser
 
 ### Security Risks of Viewing Archived JS
 
-Be aware that malicious archived JS can access the contents of other pages in your archive when viewed. Because the Web UI serves all viewed snapshots from a single domain, they share a request context and **typical CSRF/CORS/XSS/CSP protections do not work to prevent cross-site request attacks**. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) page and [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239) for more details.
+Archived JavaScript is untrusted content. The default <code>SERVER_SECURITY_MODE=auto</code> uses isolated subdomains with full replay on <code>*.localhost</code>, and a one-domain no-JS replay policy on ordinary public or LAN hostnames. Choose <code>safe-subdomains-fullreplay</code> only when wildcard DNS and TLS are configured. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview) and [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239) for details.
 
 
 <br/>
@@ -988,18 +995,16 @@ Be aware that malicious archived JS can access the contents of other pages in yo
 <summary><i>Expand to see risks and mitigations...</i></summary>
 
 
-<pre lang="bash"><code style="white-space: pre-line"># visiting an archived page with malicious JS:
-https://127.0.0.1:8000/archive/1602401954/example.com/index.html
+<pre lang="bash"><code style="white-space: pre-line"># Explicit wildcard mode: full replay on isolated snapshot subdomains
+archivebox config --set SERVER_SECURITY_MODE=safe-subdomains-fullreplay
 
-# example.com/index.js can now make a request to read everything from:
-https://127.0.0.1:8000/index.html
-https://127.0.0.1:8000/archive/*
-# then example.com/index.js can send it off to some evil server
+# Alternative for deployments without wildcard subdomains: disable JS replay
+archivebox config --set SERVER_SECURITY_MODE=safe-onedomain-nojsreplay
 </code></pre>
 
 <blockquote>
 <p><em>NOTE: Only the <code>wget</code> &amp; <code>dom</code> extractor methods execute archived JS when viewing snapshots, all other archive methods produce static output that does not execute JS on viewing.</em><br/>
-<em>If you are worried about these issues ^ you should disable these extractors using:<br/> <code>archivebox config --set SAVE_WGET=False SAVE_DOM=False</code>.</em></p>
+<em>If you do not need JavaScript-capable replay at all, you can also disable those extractors with:<br/> <code>archivebox config --set WGET_ENABLED=False DOM_ENABLED=False</code>.</em></p>
 </blockquote>
 
 <h4>Learn More</h4>
@@ -1026,7 +1031,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 <ul>
 <li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
-<li>Set up a logged-in browser session for archiving using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile"><code>CHROME_USER_DATA_DIR</code> &amp; <code>COOKIES_FILE</code></a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">importing a dedicated browser profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1039,7 +1044,7 @@ In the future we plan on adding support for running JS scripts during archiving
 
 ### Saving Multiple Snapshots of a Single URL
 
-ArchiveBox appends a hash with the current date `https://example.com#2020-10-24` to differentiate when a single URL is archived multiple times.
+ArchiveBox can preserve multiple snapshots of the same URL. The default <code>ONLY_NEW=True</code> skips URLs already in the collection; use <code>--no-only-new</code> when you intentionally want another snapshot.
 
 
 <br/>
@@ -1048,16 +1053,13 @@ ArchiveBox appends a hash with the current date `https://example.com#2020-10-24`
 <br/>
 
 
-Because ArchiveBox uniquely identifies snapshots by URL, it must use a workaround to take multiple snapshots of the same URL (otherwise they would show up as a single Snapshot entry). It makes the URLs of repeated snapshots unique by adding a hash with the archive date at the end:
+Each re-archive creates a distinct Snapshot row for the same URL:
 
-<pre lang="bash"><code style="white-space: pre-line">archivebox add 'https://example.com#2020-10-24'
-...
-archivebox add 'https://example.com#2020-10-25'
+<pre lang="bash"><code style="white-space: pre-line">archivebox add 'https://example.com'
+archivebox add --no-only-new 'https://example.com'
 </code></pre>
 
-The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI is a shortcut for this hash-date multi-snapshotting workaround.
-
-Improved support for saving multiple snapshots of a single URL without this hash-date workaround will be <a href="https://github.com/ArchiveBox/ArchiveBox/issues/179">added eventually</a> (along with the ability to view diffs of the changes between runs).
+The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI performs the same explicit re-archive.
 
 <h4>Learn More</h4>
 
@@ -1085,7 +1087,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>
 
@@ -1095,8 +1097,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
 </ul>
 
 
@@ -1185,7 +1186,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica
 
 In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.
 
-As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
+As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal counsel if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and placing branding overrides in your collection's fixed <code>custom_templates/</code> directory).
 
 </details>
 <br/>
@@ -1235,8 +1236,6 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
 
 <br/>
 
-<!--<div align="center" style="text-align: center"><br/><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/04808ac2-3133-44fd-8703-3387e06dc851" width="100%" alt="dependencies graphic"></div>-->
-
 ## Internet Archiving Ecosystem
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/78d8a725-97f4-47f5-b983-1f62843ddc51" width="14%" align="right" style="float: right"/>
@@ -1335,7 +1334,7 @@ All contributions to ArchiveBox are welcomed! Check our [issues](https://github.
 
 For low hanging fruit / easy first tickets, see: <a href="https://github.com/ArchiveBox/ArchiveBox/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22help+wanted%22">ArchiveBox/Issues `#good first ticket` `#help wanted`</a>.
 
-**Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main
+**Python API Documentation:** https://docs.archivebox.io/dev/apidocs/
 
 **Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
 
@@ -1390,7 +1389,8 @@ archivebox server 0.0.0.0:8000
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
 
 # Run the development server w/ autoreloading (but no bg workers)
 docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
@@ -1444,7 +1444,7 @@ docker run archivebox/archivebox:dev version
 ```
 
 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
 ```bash
@@ -1459,9 +1459,8 @@ services:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
 docker run -it -v $PWD:/data archivebox:dev init
 
-# or with pip:
-pip install 'git+https://github.com/pirate/ArchiveBox@dev'
-npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
+# or with uv:
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox install
 ```
 
@@ -1486,8 +1485,7 @@ archivebox install
 
 ```bash
 # generate the database migrations after changes to models.py
-cd archivebox/
-./manage.py makemigrations
+archivebox manage makemigrations
 
 # enter a python shell or a SQL shell
 cd path/to/test/data/
@@ -1496,8 +1494,7 @@ archivebox manage dbshell
 
 # generate a graph of the ORM models
 brew install graphviz
-pip install pydot graphviz
-archivebox manage graph_models -a -o orm.png
+uv run --with pydot --with graphviz archivebox manage graph_models -a -o orm.png
 open orm.png
 
 # list all models with field db info and methods
@@ -1505,7 +1502,7 @@ archivebox manage list_model_info --all --signature --db-type --field-class
 
 # print all django settings
 archivebox manage print_settings
-archivebox manage print_settings --format=yaml    # pip install pyyaml
+uv run --with pyyaml archivebox manage print_settings --format=yaml
 
 # autogenerate an admin.py from given app models
 archivebox manage admin_generator core > core/admin.py
@@ -1519,9 +1516,8 @@ archivebox manage runscript testdata
 archivebox manage reset_db
 
 # use django-tui to interactively explore commands
-uv pip install django-tui
 # ensure django-tui is in INSTALLED_APPS: core/settings.py
-archivebox manage tui
+uv run --with django-tui archivebox manage tui
 ```
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dc3e9f8c-9544-46e0-a7f0-30f571b72022" width="600px" alt="ArchiveBox ORM models relatinoship graph"/>
@@ -1559,22 +1555,6 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 
 </details>
 
-#### Roll a release
-
-<details><summary><i>Click to expand...</i></summary>
-
-(Normally CI takes care of this, but these scripts can be run to do it manually)
-```bash
-./bin/release.sh
-
-# or individually:
-./bin/release_docs.sh
-./bin/release_pip.sh
-./bin/release_docker.sh
-```
-
-</details>
-
 ---
 
 ## Further Reading

archivebox/__init__.py

@@ -38,11 +38,6 @@ ASCII_LOGO = """
 
 PACKAGE_DIR = Path(__file__).resolve().parent
 
-# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
-# # Migrations reference models like 'machine.Binary' which need to be importable
-# if str(PACKAGE_DIR) not in sys.path:
-#     sys.path.append(str(PACKAGE_DIR))
-
 os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
 os.environ["TZ"] = "UTC"
 
@@ -51,10 +46,9 @@ from .config.permissions import drop_privileges  # noqa
 
 drop_privileges()
 
-from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
+from .misc.checks import check_not_root, check_io_encoding  # noqa
 
 check_not_root()
-check_not_inside_source_dir()
 check_io_encoding()
 
 from .config.version import VERSION  # noqa
@@ -79,16 +73,10 @@ def __getattr__(name: str):
         return VERSION
     if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
         from abx_plugins import get_plugins_dir
+        from .config.constants import CONSTANTS
 
         builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = (
-            Path(
-                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
-                or os.environ.get("USER_PLUGINS_DIR")
-                or os.environ.get("DATA_DIR", os.getcwd()),
-            )
-            / "custom_plugins"
-        )
+        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
         plugins = {
             "builtin": builtin_plugins_dir,
             "user": user_plugins_dir,

archivebox/__main__.py

@@ -17,4 +17,4 @@ ASCII_LOGO_MINI = r"""
 """
 
 if __name__ == "__main__":
-    main(args=sys.argv[1:], stdin=sys.stdin)
+    main(args=sys.argv[1:])

archivebox/api/auth.py

@@ -51,6 +51,27 @@ def auth_using_token(token: str | None, request: HttpRequest | None = None) -> U
     return user
 
 
+def token_from_request(request: HttpRequest) -> str:
+    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key") or ""
+    auth_header = request.headers.get("Authorization", "")
+    if not token and auth_header.lower().startswith("bearer "):
+        token = auth_header.split(None, 1)[1].strip()
+    return token
+
+
+def authenticated_user_from_request(request: HttpRequest) -> User | None:
+    user = request.user
+    if user.is_authenticated and user.is_active:
+        return user
+
+    token = token_from_request(request)
+    token_user = auth_using_token(token=token, request=request) if token else None
+    if token_user and token_user.is_active:
+        request.user = token_user
+        return token_user
+    return None
+
+
 def auth_using_password(username: str | None, password: str | None, request: HttpRequest | None = None) -> User | None:
     """Given a username and password, check if they are valid and return the corresponding user"""
     user: User | None = None

archivebox/api/migrations/0001_initial.py

@@ -12,19 +12,9 @@ import signal_webhooks.fields
 import signal_webhooks.utils
 
 
-class Migration(migrations.Migration):
-    initial = True
-
-    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunSQL(
-                    sql="""
+# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
+# RunSQL (identical statement splitting → byte-for-byte identical behavior).
+INITIAL_SQL = """
                 -- Create api_apitoken table
                 CREATE TABLE IF NOT EXISTS api_apitoken (
                     id TEXT PRIMARY KEY NOT NULL,
@@ -70,12 +60,60 @@ class Migration(migrations.Migration):
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """,
-                    reverse_sql="""
+                    """
+
+INITIAL_REVERSE_SQL = """
                 DROP TABLE IF EXISTS api_outboundwebhook;
                 DROP TABLE IF EXISTS api_apitoken;
-                    """,
-                ),
+                    """
+
+
+def _run_sqlite_only_sql(apps, schema_editor):
+    if schema_editor.connection.vendor != "sqlite":
+        return
+    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
+        "api",
+        schema_editor,
+        None,
+        None,
+    )
+
+
+def _run_sqlite_only_sql_reverse(apps, schema_editor):
+    if schema_editor.connection.vendor != "sqlite":
+        return
+    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
+        "api",
+        schema_editor,
+        None,
+        None,
+    )
+
+
+def _pg_sync_schema(apps, schema_editor):
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
+
+
+def _pg_drop_schema(apps, schema_editor):
+    from archivebox.misc.db import drop_models_on_postgres
+
+    drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
             ],
             state_operations=[
                 migrations.CreateModel(
@@ -236,4 +274,8 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        # On non-sqlite backends the raw DDL above is skipped, so the real
+        # schema diverges from Django state. Rebuild both tables (including the
+        # unique constraint) from the post-migration state (no-op on sqlite).
+        migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
     ]

archivebox/api/urls.py

@@ -23,7 +23,4 @@ urlpatterns = [
     path("v1/", RedirectView.as_view(url="/api/v1/docs")),
     path("v1/", v1_api_urls),
     path("v1", RedirectView.as_view(url="/api/v1/docs")),
-    # ... v2 can be added here ...
-    # path("v2/",              v2_api_urls),
-    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
 ]

archivebox/api/v1_api.py

@@ -68,19 +68,16 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
 
         # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
+        api_token_attr = request.__dict__.get("_api_token")
         api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
         token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
 
-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
         response["X-ArchiveBox-Auth-Expires"] = token_expiry
         response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
-        # import ipdb; ipdb.set_trace()
-        # print('RESPONDING NOW', response)
-
         return response
 
 
@@ -91,8 +88,6 @@ api = NinjaAPIWithIOCapture(
     auth=API_AUTH_METHODS,
     urls_namespace="api-1",
     docs=Swagger(settings={"persistAuthorization": True}),
-    # docs_decorator=login_required,
-    # renderer=ORJSONRenderer(),
 )
 api = register_urls(api)
 urls = api.urls
@@ -113,24 +108,7 @@ def generic_exception_handler(request, err):
             "message": f"{err.__class__.__name__}: {err}",
             "errors": [
                 "".join(format_exception(err)),
-                # or send simpler parent-only traceback:
-                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
             ],
         },
         status=status,
     )
-
-
-# import orjson
-# from ninja.renderers import BaseRenderer
-# class ORJSONRenderer(BaseRenderer):
-#     media_type = "application/json"
-#     def render(self, request, data, *, response_status):
-#         return {
-#             "success": True,
-#             "errors": [],
-#             "result": data,
-#             "stdout": ansi_to_html(stdout.getvalue().strip()),
-#             "stderr": ansi_to_html(stderr.getvalue().strip()),
-#         }
-#         return orjson.dumps(data)

archivebox/api/v1_cli.py

@@ -8,12 +8,13 @@ from enum import Enum
 from django.http import HttpRequest
 
 from ninja import Router, Schema
+from ninja.errors import HttpError
+from pydantic import Field
 
 from archivebox.misc.util import ansi_to_html
+from archivebox.core.models import SnapshotQuerySet
 
 
-# from .auth import API_AUTH_METHODS
-
 # router for API that exposes archivebox cli subcommands as REST endpoints
 router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 
@@ -21,6 +22,7 @@ router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 # Schemas
 
 JSONType = list[Any] | dict[str, Any] | bool | int | str | None
+FILTER_PATTERNS_EXAMPLES = [["https://example.com"]]
 
 
 class CLICommandResponseSchema(Schema):
@@ -32,26 +34,11 @@ class CLICommandResponseSchema(Schema):
     stderr: str
 
 
-class FilterTypeChoices(str, Enum):
-    exact = "exact"
-    substring = "substring"
-    regex = "regex"
-    domain = "domain"
-    tag = "tag"
-    timestamp = "timestamp"
-
-
-class StatusChoices(str, Enum):
-    indexed = "indexed"
-    archived = "archived"
-    unarchived = "unarchived"
-    present = "present"
-    valid = "valid"
-    invalid = "invalid"
-    duplicate = "duplicate"
-    orphaned = "orphaned"
-    corrupted = "corrupted"
-    unrecognized = "unrecognized"
+FilterTypeChoices = Enum(
+    "FilterTypeChoices",
+    {filter_type: filter_type for filter_type in SnapshotQuerySet.FILTER_TYPE_CHOICES},
+    type=str,
+)
 
 
 class AddCommandSchema(Schema):
@@ -65,20 +52,34 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
+    persona: str = "Default"
     only_new: bool | None = None
     update: bool = False
     overwrite: bool = False
     index_only: bool = False
 
 
-class UpdateCommandSchema(Schema):
-    resume: str | None = None
+class SnapshotFilterCommandSchema(Schema):
     after: float | None = 0
-    before: float | None = 999999999999999
+    before: float | None = None
     filter_type: str | None = FilterTypeChoices.substring
-    filter_patterns: list[str] | None = ["https://example.com"]
+    filter_patterns: list[str] | None = Field(default=None, examples=FILTER_PATTERNS_EXAMPLES)
+    status: str | None = None
+    url__icontains: str | None = None
+    url__istartswith: str | None = None
+    tag: str | None = None
+    crawl_id: str | None = None
+    limit: int | None = None
+    sort: str | None = None
+    search: str | None = None
+
+
+class UpdateCommandSchema(SnapshotFilterCommandSchema):
+    resume: str | None = None
     batch_size: int = 100
     continuous: bool = False
+    index_only: bool = False
+    migrate_only: bool = False
 
 
 class ScheduleCommandSchema(Schema):
@@ -97,37 +98,43 @@ class ScheduleCommandSchema(Schema):
     clear: bool = False
 
 
-class ListCommandSchema(Schema):
-    filter_patterns: list[str] | None = ["https://example.com"]
-    filter_type: str = FilterTypeChoices.substring
-    status: StatusChoices = StatusChoices.indexed
-    after: float | None = 0
-    before: float | None = 999999999999999
-    sort: str = "bookmarked_at"
+class ListCommandSchema(SnapshotFilterCommandSchema):
     as_json: bool = True
     as_html: bool = False
     as_csv: str | None = "timestamp,url"
     with_headers: bool = False
 
 
-class RemoveCommandSchema(Schema):
-    after: float | None = 0
-    before: float | None = 999999999999999
+class RemoveCommandSchema(SnapshotFilterCommandSchema):
     filter_type: str = FilterTypeChoices.exact
-    filter_patterns: list[str] | None = ["https://example.com"]
+    timeout: float = 60.0
+
+
+def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_type: str) -> dict[str, Any]:
+    kwargs = args.dict()
+    kwargs["filter_patterns"] = kwargs.get("filter_patterns") or []
+    kwargs["filter_type"] = kwargs.get("filter_type") or default_filter_type
+    return kwargs
 
 
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
+    from archivebox.misc.util import validate_url
 
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
     if args.update or args.overwrite:
         config_overrides["ONLY_NEW"] = False
+    submitted_urls: str | list[str] = args.urls
+    if len(args.urls) == 1:
+        try:
+            validate_url(args.urls[0])
+        except ValueError:
+            submitted_urls = args.urls[0]
     crawl, snapshots = add(
-        urls=args.urls,
+        urls=submitted_urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
@@ -137,6 +144,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
         plugins=args.plugins,
+        persona=args.persona,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,
@@ -150,8 +158,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "snapshot_ids": snapshot_ids,
         "queued_urls": args.urls,
     }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
 
     return {
         "success": True,
@@ -165,24 +173,39 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
 
 @router.post("/update", response=CLICommandResponseSchema, summary="archivebox update [args] [filter_patterns]")
 def cli_update(request: HttpRequest, args: UpdateCommandSchema):
-    from archivebox.cli.archivebox_update import update
+    from archivebox.cli.archivebox_update import _build_filtered_snapshots_queryset, update
+    from archivebox.core.snapshot_status import normalize_snapshot_status
 
-    result = update(
-        filter_patterns=args.filter_patterns or [],
-        filter_type=args.filter_type or FilterTypeChoices.substring,
-        after=args.after,
-        before=args.before,
-        resume=args.resume,
-        batch_size=args.batch_size,
-        continuous=args.continuous,
-        stop_daemon_stack=False,
+    try:
+        status = normalize_snapshot_status(args.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+
+    update_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    update_kwargs["status"] = status
+    update_kwargs["stop_daemon_stack"] = False
+
+    is_filtered_update = any(
+        (update_kwargs.get(key) for key in (*SnapshotQuerySet.FILTER_ARG_KEYS, "resume") if key != "filter_type"),
     )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    matched_snapshot_ids = []
+    if is_filtered_update:
+        matched_snapshot_ids = [
+            str(snapshot_id) for snapshot_id in _build_filtered_snapshots_queryset(**update_kwargs).values_list("id", flat=True)
+        ]
+
+    update(**update_kwargs)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
-        "result": result,
+        "result": {
+            "matched_count": len(matched_snapshot_ids),
+            "snapshot_ids": matched_snapshot_ids,
+        }
+        if is_filtered_update
+        else None,
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",
         "stderr": ansi_to_html(stderr.getvalue().strip()) if isinstance(stderr, StringIO) else "",
     }
@@ -211,8 +234,8 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
         config=config_overrides or None,
     )
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -225,32 +248,38 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
 
 @router.post("/search", response=CLICommandResponseSchema, summary="archivebox search [args] [filter_patterns]")
 def cli_search(request: HttpRequest, args: ListCommandSchema):
-    from archivebox.cli.archivebox_search import search
+    from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    result = search(
-        filter_patterns=args.filter_patterns,
-        filter_type=args.filter_type,
-        status=args.status,
-        after=args.after,
-        before=args.before,
-        sort=args.sort,
-        csv=args.as_csv,
-        json=args.as_json,
-        html=args.as_html,
-        with_headers=args.with_headers,
-    )
+    search_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.substring)
+    as_json = search_kwargs.pop("as_json")
+    as_html = search_kwargs.pop("as_html")
+    as_csv = search_kwargs.pop("as_csv")
+    with_headers = search_kwargs.pop("with_headers")
+    try:
+        snapshots = build_snapshot_queryset(**search_kwargs).select_related("crawl", "crawl__created_by")
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
     result_format = "txt"
-    if args.as_json:
+    if as_json:
         result_format = "json"
-        result = json.loads(result)
-    elif args.as_html:
+        result = [
+            json.loads(json.dumps(snapshot.to_dict(extended=True), default=str))
+            for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)
+        ]
+    elif as_html:
         result_format = "html"
-    elif args.as_csv:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
+    elif as_csv:
         result_format = "csv"
+        cols = [col.strip() for col in as_csv.split(",") if col.strip()]
+        rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.prefetch_related("tags").iterator(chunk_size=500)]
+        result = "\n".join((",".join(cols), *rows) if with_headers else rows)
+    else:
+        result = "\n".join(snapshot.url for snapshot in snapshots.iterator(chunk_size=500))
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -264,37 +293,26 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
 @router.post("/remove", response=CLICommandResponseSchema, summary="archivebox remove [args] [filter_patterns]")
 def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
     from archivebox.cli.archivebox_remove import remove
-    from archivebox.cli.archivebox_search import get_snapshots
     from archivebox.core.models import Snapshot
 
-    filter_patterns = args.filter_patterns or []
-    snapshots_to_remove = get_snapshots(
-        filter_patterns=filter_patterns,
-        filter_type=args.filter_type,
-        after=args.after,
-        before=args.before,
-    )
-    removed_snapshot_ids = [str(snapshot_id) for snapshot_id in snapshots_to_remove.values_list("id", flat=True)]
+    remove_kwargs = snapshot_filter_kwargs(args, default_filter_type=FilterTypeChoices.exact)
+    timeout_arg = remove_kwargs.pop("timeout")
+    timeout = min(float(timeout_arg if timeout_arg is not None else 60.0), 60.0)
+    try:
+        snapshots_to_remove = Snapshot.objects.order_by("-created_at").search(**remove_kwargs)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
 
-    remove(
+    result = remove(
         yes=True,  # no way to interactively ask for confirmation via API, so we force yes
         snapshots=snapshots_to_remove,
-        before=args.before,
-        after=args.after,
-        filter_type=args.filter_type,
-        filter_patterns=filter_patterns,
+        timeout=timeout,
     )
-
-    result = {
-        "removed_count": len(removed_snapshot_ids),
-        "removed_snapshot_ids": removed_snapshot_ids,
-        "remaining_snapshots": Snapshot.objects.count(),
-    }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
-        "success": True,
-        "errors": [],
+        "success": bool(result["success"]),
+        "errors": [str(result["error"])] if result["error"] else [],
         "result": result,
         "result_format": "json",
         "stdout": ansi_to_html(stdout.getvalue().strip()) if isinstance(stdout, StringIO) else "",

archivebox/api/v1_core.py

@@ -4,6 +4,7 @@ import math
 import json
 import mimetypes
 import re
+from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
@@ -29,7 +30,7 @@ from ninja.errors import HttpError
 
 from archivebox.core.models import Snapshot, ArchiveResult, Tag
 from archivebox.core.permissions import public_snapshots_queryset
-from archivebox.api.auth import auth_using_token
+from archivebox.api.auth import authenticated_user_from_request
 from archivebox.config.common import get_config
 from archivebox.core.routes_util import build_web_url
 from archivebox.misc.util import filter_queryset_by_uuid_substring, validate_url_length
@@ -49,9 +50,11 @@ from archivebox.core.tag_util import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
-from archivebox.api.v1_crawls import CrawlSchema
+from archivebox.crawls.locks import crawl_lifecycle_lock
+from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
 from archivebox.search.config import get_search_mode, get_search_mode_backend
 from archivebox.search.query import apply_snapshot_search
+from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
 
 
 router = Router(tags=["Core Models"])
@@ -79,7 +82,7 @@ class CustomPagination(PaginationBase):
     def paginate_queryset(self, queryset, pagination: Input, request: HttpRequest, **params):
         limit = min(pagination.limit, 500)
         offset = pagination.offset or (pagination.page * limit)
-        total = queryset.count()
+        total = queryset.values("pk").distinct().count() if queryset.query.distinct else queryset.count()
         total_pages = math.ceil(total / limit)
         current_page = math.ceil(offset / (limit + 1))
         items = queryset[offset : offset + limit]
@@ -212,7 +215,10 @@ class ArchiveResultFilterSchema(FilterSchema):
 @paginate(CustomPagination)
 def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
     """List all ArchiveResult entries matching these filters."""
-    return filters.filter(ArchiveResult.objects.all()).distinct()
+    queryset = filters.filter(ArchiveResult.objects.all())
+    if filters.search or filters.snapshot_tag:
+        return queryset.distinct()
+    return queryset
 
 
 def _uuid_ref_query(field_name: str, ref: str) -> Q:
@@ -273,7 +279,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]
 
 
 def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = getattr(request, "_archiveresult_upload_data", None)
+    cached = request.__dict__.get("_archiveresult_upload_data")
     if cached is not None:
         return cached
 
@@ -468,12 +474,7 @@ def _write_archiveresult_files(
             )
 
         guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (
-            (mime_types[0] if mime_types else "")
-            or getattr(uploaded_file, "content_type", None)
-            or guessed_mime
-            or "application/octet-stream"
-        )
+        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
         output_files[relative_output_path] = {
             "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
             "mimetype": output_mime_type,
@@ -501,7 +502,7 @@ def _write_archiveresult_files(
         guessed_mime = mimetypes.guess_type(saved_output_path)[0]
         output_mime_type = (
             (mime_types[index] if index < len(mime_types) else "")
-            or getattr(uploaded_file, "content_type", None)
+            or uploaded_file.content_type
             or guessed_mime
             or "application/octet-stream"
         )
@@ -702,7 +703,7 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archiveresults(obj, context):
-        if bool(getattr(context["request"], "with_archiveresults", False)):
+        if bool(context["request"].__dict__.get("with_archiveresults", False)):
             return obj.archiveresult_set.all().distinct()
         return ArchiveResult.objects.none()
 
@@ -833,7 +834,7 @@ def _snapshots_rss_response(
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=snapshot.title or snapshot.url,
+            title=unescape(snapshot.title or snapshot.url),
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),
@@ -859,6 +860,7 @@ class SnapshotFilterSchema(FilterSchema):
     modified_at__lt: Annotated[datetime | None, FilterLookup("modified_at__lt")] = None
     search: str | None = None
     search_mode: str | None = None
+    status: str | None = None
     url: Annotated[str | None, FilterLookup("url")] = None
     tag: Annotated[str | None, FilterLookup("tags__name")] = None
     title: Annotated[str | None, FilterLookup("title__icontains")] = None
@@ -872,18 +874,25 @@ class SnapshotFilterSchema(FilterSchema):
     def filter_search_mode(self, value: str | None) -> Q:
         return Q()
 
+    def filter_status(self, value: str | None) -> Q:
+        return Q()
+
 
 @router.get("/snapshots", response=list[SnapshotSchema], url_name="get_snapshots")
 @paginate(CustomPagination)
 def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], with_archiveresults: bool = False):
     """List all Snapshot entries matching these filters."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = filters.filter(Snapshot.objects.all()).distinct()
+    try:
+        queryset = filter_snapshots_by_status(Snapshot.objects.all(), filters.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
+    queryset = filters.filter(queryset).distinct()
     query = (filters.search or "").strip()
     if not query:
         return queryset
 
-    runtime_config = getattr(request, "archivebox_config", None)
+    runtime_config = request.archivebox_config
     search_mode = get_search_mode(filters.search_mode, config=runtime_config)
     try:
         return apply_snapshot_search(
@@ -921,18 +930,16 @@ def get_snapshots_rss(
 def get_snapshot(request: HttpRequest, snapshot_id: str, with_archiveresults: bool = True):
     """Get a specific Snapshot by id."""
     setattr(request, "with_archiveresults", with_archiveresults)
-    queryset = Snapshot.objects.all()
-    try:
-        return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        return queryset.get(_uuid_ref_query("id", snapshot_id))
+    return _get_snapshot_by_ref(snapshot_id)
 
 
 @router.post("/snapshots", response=SnapshotSchema, url_name="create_snapshot")
 def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     tags = normalize_tag_list(data.tags)
-    if data.status is not None and data.status not in Snapshot.StatusChoices.values:
-        raise HttpError(400, f"Invalid status: {data.status}")
+    try:
+        status = normalize_snapshot_status(data.status)
+    except ValueError as err:
+        raise HttpError(400, str(err)) from err
     if not data.url.strip():
         raise HttpError(400, "URL is required")
     try:
@@ -943,7 +950,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
         raise HttpError(400, "depth must be between 0 and 4")
 
     if data.crawl_id:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), data.crawl_id).get()
+        crawl = get_crawl_by_ref(data.crawl_id)
         crawl_tags = normalize_tag_list(crawl.tags_str.split(","))
         tags = tags or crawl_tags
     else:
@@ -956,39 +963,38 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    snapshot_defaults = {
-        "depth": data.depth,
-        "title": data.title,
-        "timestamp": str(timezone.now().timestamp()),
-        "status": data.status or Snapshot.StatusChoices.QUEUED,
-        "retry_at": timezone.now(),
-    }
-    snapshot, _ = Snapshot.objects.get_or_create(
-        url=data.url,
-        crawl=crawl,
-        defaults=snapshot_defaults,
-    )
+    with crawl_lifecycle_lock(str(crawl.id)):
+        snapshot_defaults = {
+            "depth": data.depth,
+            "title": data.title,
+            "timestamp": str(timezone.now().timestamp()),
+            "status": status or Snapshot.StatusChoices.QUEUED,
+            "retry_at": timezone.now(),
+        }
+        snapshot, _ = Snapshot.objects.get_or_create(
+            url=data.url,
+            crawl=crawl,
+            defaults=snapshot_defaults,
+        )
 
-    update_fields: list[str] = []
-    if data.title is not None and snapshot.title != data.title:
-        snapshot.title = data.title
-        update_fields.append("title")
-    if data.status is not None and snapshot.status != data.status:
-        if data.status not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {data.status}")
-        snapshot.status = data.status
-        update_fields.append("status")
-    if update_fields:
-        update_fields.append("modified_at")
-        snapshot.save(update_fields=update_fields)
+        update_fields: list[str] = []
+        if data.title is not None and snapshot.title != data.title:
+            snapshot.title = data.title
+            update_fields.append("title")
+        if status is not None and snapshot.status != status:
+            snapshot.status = status
+            update_fields.append("status")
+        if update_fields:
+            update_fields.append("modified_at")
+            snapshot.save(update_fields=update_fields)
 
-    if tags:
-        snapshot.save_tags(tags)
+        if tags:
+            snapshot.save_tags(tags)
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
+        try:
+            snapshot.ensure_crawl_symlink()
+        except Exception:
+            pass
 
     setattr(request, "with_archiveresults", False)
     return snapshot
@@ -997,10 +1003,8 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 @router.patch("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="patch_snapshot")
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
-    try:
-        snapshot = Snapshot.objects.get(Q(id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
-    except Snapshot.DoesNotExist:
-        snapshot = filter_queryset_by_uuid_substring(Snapshot.objects.all(), snapshot_id).get()
+    snapshot = _get_snapshot_by_ref(snapshot_id)
+    crawl_id = str(snapshot.crawl_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -1022,25 +1026,28 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
             return snapshot
         raise HttpError(400, f"Invalid action: {action}")
 
-    if "status" in payload:
-        if payload["status"] not in Snapshot.StatusChoices.values:
-            raise HttpError(400, f"Invalid status: {payload['status']}")
-        snapshot.status = payload["status"]
-        if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-            snapshot.retry_at = None
-        update_fields.append("status")
+    with crawl_lifecycle_lock(crawl_id):
+        snapshot = _get_snapshot_by_ref(snapshot_id)
+        if "status" in payload:
+            try:
+                snapshot.status = normalize_snapshot_status(payload["status"])
+            except ValueError as err:
+                raise HttpError(400, str(err)) from err
+            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
+                snapshot.retry_at = None
+            update_fields.append("status")
 
-    if "retry_at" in payload:
-        snapshot.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
+        if "retry_at" in payload:
+            snapshot.retry_at = payload["retry_at"]
+            update_fields.append("retry_at")
 
-    if tags is not None:
-        snapshot.save_tags(normalize_tag_list(tags))
+        if tags is not None:
+            snapshot.save_tags(normalize_tag_list(tags))
 
-    if payload.get("status") == Snapshot.StatusChoices.SEALED:
-        snapshot.cancel()
-    else:
-        snapshot.save(update_fields=update_fields)
+        if payload.get("status") == Snapshot.StatusChoices.SEALED:
+            snapshot.cancel()
+        else:
+            snapshot.save(update_fields=update_fields)
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1050,7 +1057,14 @@ def delete_snapshot(request: HttpRequest, snapshot_id: str):
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
     crawl_id_str = str(snapshot.crawl.pk)
-    deleted_count, _ = snapshot.delete()
+    snapshot.cancel()
+
+    from archivebox.services.runner import run_pending_crawls
+
+    with crawl_lifecycle_lock(crawl_id_str):
+        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
+        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
+        deleted_count, _ = snapshot.delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
@@ -1081,7 +1095,7 @@ class TagSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -1090,7 +1104,7 @@ class TagSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -1294,16 +1308,7 @@ def _public_tag_listing_enabled() -> bool:
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = getattr(request, "user", None)
-    if getattr(user, "is_authenticated", False):
-        return True
-
-    token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-    auth_header = request.headers.get("Authorization", "")
-    if not token and auth_header.lower().startswith("bearer "):
-        token = auth_header.split(None, 1)[1].strip()
-
-    if token and auth_using_token(token=token, request=request):
+    if authenticated_user_from_request(request):
         return True
 
     return _public_tag_listing_enabled()
@@ -1315,7 +1320,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
+    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
     queryset = get_matching_tags(q)
     public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
     if public_only:
@@ -1324,7 +1329,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
     }
 
 

archivebox/api/v1_crawls.py

@@ -1,5 +1,6 @@
 __package__ = "archivebox.api"
 
+import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -24,7 +25,7 @@ from archivebox.config.common import get_config
 from archivebox.crawls.models import Crawl
 from archivebox.misc.util import filter_queryset_by_uuid_substring
 
-from .auth import API_AUTH_METHODS, auth_using_token
+from .auth import API_AUTH_METHODS, authenticated_user_from_request
 
 router = Router(tags=["Crawl Models"], auth=API_AUTH_METHODS)
 
@@ -48,8 +49,6 @@ class CrawlSchema(Schema):
     tags_str: str
     config: dict
 
-    # snapshots: List[SnapshotSchema]
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -58,7 +57,7 @@ class CrawlSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -71,7 +70,7 @@ class CrawlSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -122,10 +121,9 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    request_user = request.user if request.user.is_authenticated else None
-    config.setdefault("PERMISSIONS", str(get_config(user=request_user).PERMISSIONS))
+    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
-        urls="\n".join(urls),
+        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
         max_depth=data.max_depth,
         tags_str=",".join(tags),
         label=data.label,
@@ -138,12 +136,16 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     return crawl
 
 
+def get_crawl_by_ref(crawl_id: str):
+    return filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+
+
 @router.get("/crawl/{crawl_id}", response=CrawlSchema, url_name="get_crawl")
 def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_snapshots: bool = False, with_archiveresults: bool = False):
     """Get a specific Crawl by id."""
     setattr(request, "with_snapshots", with_snapshots)
     setattr(request, "with_archiveresults", with_archiveresults)
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
 
     if crawl and as_rss:
         query = request.GET.copy()
@@ -157,33 +159,18 @@ def get_crawl(request: HttpRequest, crawl_id: str, as_rss: bool = False, with_sn
 def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     # Try to resolve the crawl first; if it doesn't exist, return 404.
     try:
-        crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+        crawl = get_crawl_by_ref(crawl_id)
     except Crawl.DoesNotExist:
         raise HttpError(404, "Crawl not found")
 
-    # Determine the effective viewer: session user takes precedence, otherwise
-    # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
-    # Authorization: Bearer ... (so that programmatic clients still work).
-    user = getattr(request, "user", None)
-    is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False))
-    if not is_authenticated:
-        token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
-        auth_header = request.headers.get("Authorization", "")
-        if not token and auth_header.lower().startswith("bearer "):
-            token = auth_header.split(None, 1)[1].strip()
-        token_user = auth_using_token(token=token, request=request) if token else None
-        if token_user and token_user.is_active:
-            user = token_user
-            is_authenticated = True
-            # Re-bind so is_admin_user() / ownership checks below see the token user.
-            setattr(request, "user", token_user)
+    user = authenticated_user_from_request(request)
 
     # Gate access using the same model as SnapshotView/can_view_snapshot:
     # admins always pass; owners can see their own crawls; otherwise the crawl
     # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
     if not is_admin_user(request):
         permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None))
+        is_owner = bool(user and crawl.created_by_id == user.id)
         if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
             raise HttpError(404, "Crawl not found")
 
@@ -218,7 +205,7 @@ def crawl_file_nested_2(request: HttpRequest, crawl_id: str, folder: str, subfol
 @router.patch("/crawl/{crawl_id}", response=CrawlSchema, url_name="patch_crawl")
 def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
     """Update a crawl (e.g., set status=sealed to cancel queued work)."""
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
 
@@ -260,10 +247,18 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 
 @router.delete("/crawl/{crawl_id}", response=CrawlDeleteResponseSchema, url_name="delete_crawl")
 def delete_crawl(request: HttpRequest, crawl_id: str):
-    crawl = filter_queryset_by_uuid_substring(Crawl.objects.all(), crawl_id).get()
+    crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
-    snapshot_count = crawl.snapshot_set.count()
-    deleted_count, _ = crawl.delete()
+    crawl.cancel()
+
+    from archivebox.crawls.locks import crawl_lifecycle_lock
+    from archivebox.services.runner import run_pending_crawls
+
+    with crawl_lifecycle_lock(crawl_id_str):
+        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
+        crawl = get_crawl_by_ref(crawl_id_str)
+        snapshot_count = crawl.snapshot_set.count()
+        deleted_count, _ = crawl.delete()
     return {
         "success": True,
         "crawl_id": crawl_id_str,

archivebox/api/v1_personas.py

@@ -8,8 +8,10 @@ from uuid import UUID
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.pagination import paginate
 from pydantic import Field
 
+from archivebox.api.v1_core import CustomPagination
 from archivebox.personas.importers import validate_persona_name
 from archivebox.personas.models import Persona
 
@@ -113,6 +115,7 @@ def find_persona(extension_persona_id: str, name: str) -> Persona | None:
 
 
 @router.get("/personas", response=list[PersonaSchema], url_name="get_personas")
+@paginate(CustomPagination)
 def get_personas(request: HttpRequest):
     """List personas available on this ArchiveBox server."""
     return Persona.objects.all().order_by("name")
@@ -136,7 +139,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     created = persona is None
     if persona is None:
         persona = Persona(name=name)
-        if getattr(request.user, "is_authenticated", False):
+        if request.user.is_authenticated:
             persona.created_by = request.user
 
     persona.config = {

archivebox/base_models/admin.py

@@ -5,11 +5,11 @@ __package__ = "archivebox.base_models"
 import json
 import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict
+from typing import ClassVar, NotRequired, TypedDict, cast
 
 from django import forms
 from django.contrib import admin
-from django.db import models
+from django.db import DatabaseError, models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
 from django.urls import path, register_converter
@@ -68,49 +68,33 @@ class KeyValueWidget(forms.Widget):
     template_name = ""  # We render manually
 
     class Media:
-        css = {
+        css: ClassVar[dict[str, list[str]]] = {
             "all": [],
         }
-        js = []
+        js: ClassVar[list[str]] = []
 
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
         try:
-            from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.plugins.discovery import discover_plugin_configs
+            from archivebox.config.common import config_field_metadata
 
             options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}
-            for key, field in ArchiveBoxConfig.model_fields.items():
-                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
-                    continue
-                default = field.default
-                try:
-                    json.dumps(default)
-                except TypeError:
-                    default = str(default)
-                options[key] = {
-                    "plugin": "archivebox",
-                    "type": str(field.annotation),
-                    "default": default,
-                    "description": field.description or "",
+            for key, metadata in config_field_metadata().items():
+                option_type = metadata.get("type", "string")
+                option: ConfigOption = {
+                    "plugin": str(metadata.get("plugin", "archivebox")),
+                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
+                    "default": metadata.get("default", ""),
+                    "description": str(metadata.get("description", "")),
                 }
-
-            plugin_configs = discover_plugin_configs()
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
+                schema = metadata.get("schema")
+                if isinstance(schema, Mapping):
                     for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
+                        if schema_key in schema:
+                            option[schema_key] = schema[schema_key]
+                options[key] = option
             return options
-        except Exception:
+        except (ImportError, KeyError, TypeError, ValueError):
             return {}
 
     def _parse_value(self, value: object) -> dict[str, object]:
@@ -660,41 +644,64 @@ class KeyValueWidget(forms.Widget):
 
                     window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
 
-                    function focusConfigKeyFromHash_{widget_id}() {{
-                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
-                        // to (or creates) the matching row in this editor. Used by
-                        // the in-banner "pin via admin" link and the
-                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
-                        // config detail page.
-                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
-                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
-                            return;
-                        }}
+                    function configRowForKey_{widget_id}(key) {{
                         var container = document.getElementById('{widget_id}_rows');
                         if (!container) {{
-                            return;
+                            return null;
                         }}
                         var match = null;
                         container.querySelectorAll('.key-value-row').forEach(function(row) {{
                             if (match) {{ return; }}
                             var keyInput = row.querySelector('.kv-key');
-                            if (keyInput && keyInput.value.trim() === hash) {{
+                            if (keyInput && keyInput.value.trim() === key) {{
                                 match = row;
                             }}
                         }});
                         if (!match) {{
-                            // No existing row for this key — prepopulate one with the
-                            // key filled in but value left blank so the operator just
-                            // types/pastes the value and hits save.
                             window.addKeyValueRow_{widget_id}();
                             var rows = container.querySelectorAll('.key-value-row');
                             match = rows[rows.length - 1];
                             var keyInput = match.querySelector('.kv-key');
                             if (keyInput) {{
-                                keyInput.value = hash;
+                                keyInput.value = key;
                                 keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
                             }}
                         }}
+                        return match;
+                    }}
+
+                    function prefillConfigFromQuery_{widget_id}() {{
+                        var params = new URLSearchParams(window.location.search);
+                        var consumedConfigKeys = [];
+                        params.forEach(function(value, key) {{
+                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
+                                return;
+                            }}
+                            consumedConfigKeys.push(key);
+                            var match = configRowForKey_{widget_id}(key);
+                            var valueInput = match && match.querySelector('.kv-value');
+                            if (valueInput) {{
+                                valueInput.value = value;
+                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }});
+                        updateHiddenField_{widget_id}();
+                        if (consumedConfigKeys.length) {{
+                            var cleanUrl = new URL(window.location.href);
+                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
+                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
+                        }}
+                    }}
+
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the setup wizard and live-config detail pages.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var match = configRowForKey_{widget_id}(hash);
                         if (!match) {{
                             return;
                         }}
@@ -716,12 +723,14 @@ class KeyValueWidget(forms.Widget):
                     // Initialize on load
                     document.addEventListener('DOMContentLoaded', function() {{
                         initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
                         focusConfigKeyFromHash_{widget_id}();
                     }});
                     // Also run immediately in case DOM is already ready
                     if (document.readyState !== 'loading') {{
                         initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
                         focusConfigKeyFromHash_{widget_id}();
                     }}
@@ -839,10 +848,10 @@ class ConfigEditorMixin(admin.ModelAdmin):
         """
         from archivebox.config.common import is_sensitive_config_key
 
-        if change and obj.pk and getattr(obj, "config", None) is not None:
+        if change and obj.pk and obj.config is not None:
             try:
                 stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
-            except Exception:
+            except (AttributeError, DatabaseError, TypeError, ValueError):
                 stored = {}
             if isinstance(stored, dict):
                 new_config = dict(obj.config or {})
@@ -861,6 +870,20 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     list_display = ("id", "created_at", "created_by")
     readonly_fields = ("id", "created_at", "modified_at")
     show_search_mode_selector = False
+    change_form_template = "admin/archivebox_change_form.html"
+
+    def get_admin_toolbar_actions(self, request, obj):
+        """Return extra action button dicts for the shared change-form toolbar.
+
+        See ``templates/admin/includes/archivebox_toolbar.html`` for the
+        accepted keys. Default: no extras (toolbar renders Save/History/Delete
+        plus any ``django-object-actions`` change_actions).
+        """
+        return []
+
+    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
+        context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
+        return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
 
     def get_default_search_mode(self) -> str:
         # The shared changelist template always asks every admin for a default
@@ -893,9 +916,9 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
         """
         info = self.opts.app_label, self.opts.model_name
         object_routes = [
-            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
-            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
-            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
+            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
+            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
+            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
         ]
         # Append after super().get_urls() so our patterns are the
         # *last-registered* ones with the canonical admin URL names — Django's

archivebox/base_models/apps.py

@@ -1,7 +0,0 @@
-# from django.apps import AppConfig
-
-
-# class BaseModelsConfig(AppConfig):
-#     default_auto_field = 'django.db.models.BigAutoField'
-
-#     name = 'base_models'

archivebox/base_models/models.py

@@ -20,6 +20,8 @@ from django.conf import settings
 
 from django_stubs_ext.db.models import TypedModelMeta
 
+from archivebox.config import CONSTANTS
+
 
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
@@ -113,6 +115,10 @@ class ModelWithHealthStats(models.Model):
     class Meta(TypedModelMeta):
         abstract = True
 
+    @property
+    def admin_change_url(self) -> str:
+        return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
+
     @property
     def health(self) -> int:
         total = max(self.num_uses_failed + self.num_uses_succeeded, 1)
@@ -165,7 +171,7 @@ class ModelWithDeleteAfter(models.Model):
     def get_delete_after_config_value(self):
         from archivebox.config.common import get_config
 
-        return get_config(include_machine=False).DELETE_AFTER
+        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
 
     def set_delete_at_from_config(self, config_value=None) -> bool:
         if self.delete_at is not None:
@@ -253,9 +259,7 @@ class ModelWithOutputDir(ModelWithUUID):
 
     @classmethod
     def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        from archivebox.config.common import get_config
-
-        data_dir = get_config().DATA_DIR.resolve()
+        data_dir = CONSTANTS.DATA_DIR.resolve()
         safe_paths = []
         for raw_path in paths:
             path = Path(raw_path)

archivebox/cli/__init__.py

@@ -18,10 +18,11 @@ if "--debug" in sys.argv:
 
 # Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
 # `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. Strip it from argv here so each subcommand's own click parser
-# never sees it. Ignored for `help` and `init` themselves.
-if "--init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
+# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
+# `init` themselves.
+if "--init" in sys.argv or "--quick-init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
     os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
 
 
@@ -32,6 +33,7 @@ class ArchiveBoxGroup(click.Group):
         "help": "archivebox.cli.archivebox_help.main",
         "version": "archivebox.cli.archivebox_version.main",
         "mcp": "archivebox.cli.archivebox_mcp.main",
+        "oneshot": "archivebox.cli.archivebox_oneshot.main",
     }
     setup_commands = {
         "init": "archivebox.cli.archivebox_init.main",
@@ -66,10 +68,6 @@ class ArchiveBoxGroup(click.Group):
         # Introspection commands
         "pluginmap": "archivebox.cli.archivebox_pluginmap.main",
     }
-    legacy_model_commands = {
-        "crawl": "archivebox.cli.archivebox_crawl_compat.main",
-        "snapshot": "archivebox.cli.archivebox_snapshot_compat.main",
-    }
     all_subcommands = {
         **meta_commands,
         **setup_commands,
@@ -81,34 +79,29 @@ class ArchiveBoxGroup(click.Group):
         "import": "add",
         "archive": "add",
     }
-    legacy_model_subcommands = {
-        "crawl": {"create", "list", "update", "delete"},
-        "snapshot": {"create", "list", "update", "delete"},
-    }
 
     @classmethod
     def get_canonical_name(cls, cmd_name):
         return cls.renamed_commands.get(cmd_name, cmd_name)
 
     @classmethod
-    def _should_use_legacy_model_command(cls, cmd_name: str) -> bool:
-        if cmd_name not in cls.legacy_model_commands:
-            return False
+    def _needs_django_for_lazy_import(cls, cmd_name: str) -> bool:
+        wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
+        return not wants_help and (cmd_name in cls.archive_commands or cmd_name in cls.model_commands)
 
-        try:
-            arg_idx = sys.argv.index(cmd_name)
-        except ValueError:
-            return False
+    @classmethod
+    def _setup_django_for_lazy_import(cls, cmd_name: str) -> None:
+        if not cls._needs_django_for_lazy_import(cmd_name):
+            return
 
-        remaining_args = sys.argv[arg_idx + 1 :]
-        if not remaining_args:
-            return False
+        from django.apps import apps
 
-        first_arg = remaining_args[0]
-        if first_arg in ("-h", "--help"):
-            return False
+        if apps.ready:
+            return
 
-        return first_arg not in cls.legacy_model_subcommands[cmd_name]
+        from archivebox.config.django import setup_django
+
+        setup_django()
 
     def get_command(self, ctx, cmd_name):
         # handle renamed commands
@@ -120,11 +113,9 @@ class ArchiveBoxGroup(click.Group):
             cmd_name = new_name
             ctx.invoked_subcommand = cmd_name
 
-        if self._should_use_legacy_model_command(cmd_name):
-            return self._lazy_load(self.legacy_model_commands[cmd_name])
-
         # handle lazy loading of commands
         if cmd_name in self.all_subcommands:
+            self._setup_django_for_lazy_import(cmd_name)
             return self._lazy_load(cmd_name)
 
         # fall-back to using click's default command lookup
@@ -137,16 +128,12 @@ class ArchiveBoxGroup(click.Group):
             import_path = cmd_name_or_path
         modname, funcname = import_path.rsplit(".", 1)
 
-        # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
-        func = getattr(mod, funcname)
+        func = vars(mod)[funcname]
 
-        if not hasattr(func, "__doc__"):
+        if func.__doc__ is None:
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
-        # if not isinstance(cmd, click.BaseCommand):
-        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
-
         return func
 
 
@@ -167,7 +154,6 @@ def cli(ctx, help=False):
     # then we need to set up the django environment and check that we're in a valid data folder
     wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
     if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
-        # print('SETUP DJANGO AND CHECK DATA FOLDER')
         try:
             if subcommand == "server":
                 run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")
@@ -199,15 +185,12 @@ def cli(ctx, help=False):
                 raise
 
 
-def main(args=None, prog_name=None, stdin=None):
+def main(args=None, prog_name=None):
     # show `docker run archivebox xyz` in help messages if running in docker
     IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
     IS_TTY = sys.stdin.isatty()
     prog_name = prog_name or (f"docker compose run{'' if IS_TTY else ' -T'} archivebox" if IN_DOCKER else "archivebox")
 
-    # stdin param allows passing input data from caller (used by __main__.py)
-    # currently not used by click-based CLI, but kept for backwards compatibility
-
     previous_unraisablehook = sys.unraisablehook
 
     def ignore_shutdown_unraisable(unraisable):

archivebox/cli/archivebox_add.py

@@ -5,14 +5,15 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
-import json
 import os
+import json
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
 
@@ -23,21 +24,28 @@ if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err
 
     return urls
 
@@ -82,7 +90,6 @@ def add(
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox import CONSTANTS
     from archivebox.config.permissions import USER, HOSTNAME
     from archivebox.config.common import get_config
 
@@ -113,31 +120,30 @@ def add(
     from archivebox.misc.logging_util import printable_filesize
     from archivebox.misc.system import get_dir_size
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from django.contrib.auth import get_user_model
     from django.utils import timezone
 
     created_by_id = created_by_id or get_or_create_system_user_pk()
-    created_by = get_user_model().objects.filter(pk=created_by_id).first()
     started_at = timezone.now()
 
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if snapshot_ids:
-        sources_file.write_text(
-            "\n".join(json.dumps({"url": url, "id": snapshot_ids[index], "tags": tag, "depth": 0}) for index, url in enumerate(url_list)),
-        )
-    else:
-        sources_file.write_text("\n".join(url_list))
+    use_internal_input_root = isinstance(urls, str)
+    source_text = (
+        urls
+        if use_internal_input_root
+        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
+    )
 
     # 2. Create a new Crawl with inline URLs
+    # Foreground add must claim runner ownership before publishing runnable
+    # work. Otherwise an existing server runner can lease the new Crawl in the
+    # gap between Crawl.objects.create() and current_command(), then be killed
+    # by the newer add process halfway through its first Snapshot hook.
+    command = None
+    if not bg and not index_only:
+        from archivebox.machine.models import Process
+        from archivebox.core.takeover_util import current_command
+
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR)
+
     cli_args = [*sys.argv]
     if cli_args[0].lower().endswith("archivebox"):
         cli_args[0] = "archivebox"
@@ -145,13 +151,11 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    # Read URLs directly into crawl
-    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj = Persona.get_or_create_named(persona_name)
     persona_obj.ensure_dirs()
-    effective_persona_config = get_config(persona=persona_obj, user=created_by)
+    effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
         "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
@@ -172,25 +176,41 @@ def add(
         **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
     }
     # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority — they win over persona/plugin/env defaults and get stamped
-    # directly onto crawl.config so the runtime resolution and admin UI both
-    # reflect them faithfully.
+    # priority for crawl-frozen keys. Runtime-derived execution keys are
+    # stripped by Crawl.save() and rederived when hooks run.
     crawl_config.update(config_overrides)
 
-    crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
-        tags_str=tag,
-        persona_id=persona_obj.id,
-        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-        created_by_id=created_by_id,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=None if (index_only or bg) else timezone.now(),
-        config=crawl_config,
-    )
+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
+    try:
+        crawl = Crawl.objects.create(
+            urls=source_text,
+            # Stdin/import text gets an extra hop because the synthetic
+            # archivebox://internal root lives at depth 0 and parser-discovered
+            # URLs land at depth 1; direct URL args become the depth=0 input
+            # snapshots themselves so --depth=N matches the deepest hop the user
+            # asked for.
+            max_depth=depth + 1 if use_internal_input_root else depth,
+            tags_str=tag,
+            persona_id=persona_obj.id,
+            label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
+            created_by_id=created_by_id,
+            status=Crawl.StatusChoices.QUEUED,
+            retry_at=None if index_only else timezone.now(),
+            config=crawl_config,
+        )
+        first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
+    except BaseException:
+        if command is not None:
+            command.mark_exited(exit_code=1)
+        raise
+    if command is not None:
+        command.url = first_url
+        command.save(update_fields=["url", "modified_at"])
 
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
-    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
     print(f"    [dim]First URL: {first_url}[/dim]")
 
     # 3. The CrawlMachine will create Snapshots from all URLs when started
@@ -213,28 +233,43 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
+        from archivebox.services.runner import ensure_background_runner
+
+        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_runtime_stack, current_command, standby_until_runtime_stack_needed
-        from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
+        from archivebox.core.takeover_util import command_owns_foreground_runner, standby_until_foreground_runner_needed
+        from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
 
-        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
+        assert command is not None
         exit_code = 0
         try:
             try:
                 with foreground_shutdown_signals(first_signal_message=None), foreground_parent_watchdog():
                     while True:
-                        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
                         exit_code = run_runner_worker(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
                             interactive_interrupts=True,
+                            config=get_config(crawl=crawl),
                         )
+                        crawl.refresh_from_db(fields=["status", "retry_at"])
+                        if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
+                            break
+                        # A shared supervisord can be stopped by another
+                        # foreground owner (for example `archivebox server`
+                        # shutting down) and stop the one-shot add runner while
+                        # this add's crawl is still runnable. Keep the add
+                        # process alive so it can re-enter the normal
+                        # foreground-runner ownership loop and finish its crawl.
+                        supervisor_gone = exit_code == 1 and get_existing_supervisord_process(quiet=True) is None
+                        if crawl.status in crawl.RUNNABLE_STATES and (exit_code in (0, 130, 143) or supervisor_gone):
+                            continue
                         if exit_code == 0:
                             break
-                        if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+                        if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                             continue
                         raise SystemExit(exit_code)
             except KeyboardInterrupt:
@@ -315,6 +350,7 @@ def add(
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
 @click.option(
     "--only-new/--no-only-new",
@@ -336,7 +372,12 @@ def main(**kwargs):
 
     with foreground_shutdown_signals(), foreground_parent_watchdog():
         raw_urls = kwargs.pop("urls")
-        urls = _collect_input_urls(raw_urls)
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
         if not urls:
             raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
         if int(kwargs.get("max_urls") or 0) < 0:
@@ -353,16 +394,33 @@ def main(**kwargs):
             raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
         if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
             raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+        if not kwargs.get("index_only"):
+            from archivebox.workers.supervisord_util import require_crawl_memory
+
+            require_crawl_memory()
 
         # Translate --only-new/--no-only-new into a crawl config override.
         # add() takes config overrides as a dict; no per-flag kwargs.
         overwrite = kwargs.pop("overwrite", False)
         update = kwargs.pop("update", False)
         only_new = kwargs.pop("only_new", None)
+        extract = kwargs.pop("extract", "")
         if overwrite or update:
             only_new = False
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+        if extract:
+            from abx_dl.models import discover_plugins, plugins_matching_output
+
+            all_plugins = discover_plugins()
+            tokens = [token.strip() for token in extract.split(",") if token.strip()]
+            plugin_names = {name.lower(): name for name in all_plugins}
+            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
+            selected += plugins_matching_output(all_plugins, tokens)
+            if not selected:
+                raise click.UsageError(f"No plugins found matching extract types: {extract}")
+            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
+            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
         add(urls=urls, **kwargs)
 

archivebox/cli/archivebox_config.py

@@ -6,6 +6,7 @@ import sys
 import toml
 import rich_click as click
 from rich import print
+from pathlib import Path
 
 from archivebox.misc.util import docstring, enforce_types
 from archivebox.misc.toml_util import CustomTOMLEncoder
@@ -30,12 +31,17 @@ def config(
     from archivebox.misc.logging_util import printable_config
     from abx_plugins.plugins.base.utils import resolve_alias
     from archivebox.config.collection import write_config_file
+    from archivebox.config import CONSTANTS_CONFIG
     from archivebox.config.common import ArchiveBoxConfig, get_config, get_all_configs
     from archivebox.plugins.discovery import discover_plugin_configs
 
     check_data_folder()
 
     FLAT_CONFIG = get_config().as_dict()
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys()
+    readonly_config = {key: val for key, val in CONSTANTS_CONFIG.items() if key.isupper() and isinstance(val, Path)}
+    writable_config = {key: val for key, val in FLAT_CONFIG.items() if key not in runtime_derived_keys and key not in readonly_config}
+    readable_config = {**writable_config, **readonly_config}
     CONFIGS = get_all_configs()
     plugin_schemas = {
         plugin_name: schema.get("properties", {}) for plugin_name, schema in discover_plugin_configs().items() if isinstance(schema, dict)
@@ -53,23 +59,20 @@ def config(
     matching_config = {}
     if search:
         if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
+            search_terms = [key.strip().lower() for key in config_options]
 
-                for search_key in config_options:
-                    # search all aliases in the section
-                    for alias_key, key in aliases.items():
-                        if search_key.lower() in alias_key.lower():
-                            matching_config[key] = dict(config_section)[key]
+            for existing_key, value in readable_config.items():
+                if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
+                    matching_config[existing_key] = value
 
-                    # search all keys and values in the section
-                    for existing_key, value in dict(config_section).items():
-                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
-                            matching_config[existing_key] = value
+            for alias, key in core_config_aliases.items():
+                if key in readable_config and any(term in alias.lower() for term in search_terms):
+                    matching_config[key] = readable_config[key]
+
+            for schema in plugin_schemas.values():
+                for key, metadata in schema.items():
+                    if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
+                        matching_config[key] = readable_config[key]
 
         print(printable_config(matching_config))
         raise SystemExit(not matching_config)
@@ -79,18 +82,18 @@ def config(
             config_options = [
                 core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
             ]
-            matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
-            failed_config = [key for key in config_options if key not in FLAT_CONFIG]
+            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
+            failed_config = [key for key in config_options if key not in readable_config]
             if failed_config:
                 print("\n[red][X] These options failed to get[/red]")
                 print("    {}".format("\n    ".join(config_options)))
                 raise SystemExit(1)
         else:
-            matching_config = FLAT_CONFIG
+            matching_config = readable_config
 
         # Display core config sections
         for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
+            section_header = config_section.toml_section_header
             if isinstance(section_header, str) and section_header:
                 print(f"[grey53]\\[{section_header}][/grey53]")
             else:
@@ -100,12 +103,18 @@ def config(
             print(_format_toml(kv_in_section))
             print("[grey53]################################################################[/grey53]")
 
+        readonly_keys = {key: val for key, val in readonly_config.items() if key in matching_config}
+        if readonly_keys:
+            print("[grey53]\\[CONSTANTS]                                        # (read-only)[/grey53]")
+            print(_format_toml(readonly_keys))
+            print("[grey53]################################################################[/grey53]")
+
         plugin_keys = {}
 
         # Collect all plugin config keys
         for schema in plugin_schemas.values():
             for key in schema.keys():
-                if key in matching_config:
+                if key in matching_config and key in writable_config:
                     plugin_keys[key] = matching_config[key]
 
         # Display all plugin config in single [PLUGINS] section
@@ -135,7 +144,7 @@ def config(
                     f"[yellow][i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.[/yellow]",
                 )
 
-            if key in FLAT_CONFIG:
+            if key in writable_config:
                 new_config[key] = val.strip()
             else:
                 failed_options.append(line)

archivebox/cli/archivebox_crawl.py

@@ -65,6 +65,7 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -98,7 +99,11 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1
 
     # Output pass-through records first
     if not is_tty:

archivebox/cli/archivebox_crawl_compat.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox crawl"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_add import add
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--depth", "-d", type=int, default=0, help="Max crawl depth (default: 0)")
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--wait/--no-wait", "wait", default=True, help="Accepted for backwards compatibility")
-@click.argument("urls", nargs=-1)
-def main(depth: int, tag: str, status: str, wait: bool, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox crawl URL...` entrypoint."""
-    del status, wait
-    add(list(urls), depth=depth, tag=tag, bg=True)
-    sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_extract.py

@@ -46,12 +46,12 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     through the shared crawl runner with the corresponding plugin selected.
     """
     from rich import print as rprint
-    from django.utils import timezone
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
 
     try:
-        archiveresult = ArchiveResult.objects.get(id=archiveresult_id)
+        archiveresult = ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
     except ArchiveResult.DoesNotExist:
         rprint(f"[red]ArchiveResult {archiveresult_id} not found[/red]", file=sys.stderr)
         return 1
@@ -59,22 +59,9 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
     try:
-        was_paused = archiveresult.snapshot.is_paused
         archiveresult.reset_for_retry()
         snapshot = archiveresult.snapshot
-        if not was_paused:
-            snapshot.queue_for_extraction()
-        else:
-            # A paused snapshot may still accept explicit maintenance for one
-            # ArchiveResult, but this path must not transition it back to
-            # queued/startable work. Guard: only set retry_at while the row is
-            # still paused — concurrent resume would otherwise see a stale
-            # retry_at marker.
-            snapshot.safe_update(
-                {"retry_at": timezone.now()},
-                refresh=False,
-                extra_filter={"status": snapshot.StatusChoices.PAUSED},
-            )
+        snapshot.queue_for_extraction()
         crawl = snapshot.crawl
         if not crawl.claim_processing_lock(lock_seconds=10):
             rprint(
@@ -83,11 +70,7 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
             )
             return 1
 
-        try:
-            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        finally:
-            if was_paused:
-                snapshot.restore_paused_scheduler_marker()
+        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -116,6 +99,7 @@ def run_plugins(
     wait: bool = True,
     emit_results: bool = True,
     show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
     """
     Run plugins on Snapshots from input.
@@ -209,7 +193,7 @@ def run_plugins(
             if snapshot_id in existing_snapshot_ids
             for plugin_name in plugin_names
         )
-    plugins_by_name = discover_plugins()
+    plugins_by_name = discover_plugins(runtime="archivebox")
     requested_rows: set[tuple[str, str, str]] = set()
     for snapshot_id, plugin_name in requested_pairs:
         plugin = plugins_by_name.get(plugin_name)
@@ -219,6 +203,18 @@ def run_plugins(
         else:
             requested_rows.add((snapshot_id, plugin_name, ""))
 
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
+
     reset_fields = {
         "status": ArchiveResult.StatusChoices.QUEUED,
         "output_str": "",
@@ -230,23 +226,34 @@ def run_plugins(
         "end_ts": None,
         "modified_at": timezone.now(),
     }
-    if plugins_list:
-        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
-    elif requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
-            if snapshot_id in existing_snapshot_ids:
-                for plugin_name in plugin_names:
-                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
-    existing_rows = set(
-        ArchiveResult.objects.filter(
-            snapshot_id__in=existing_snapshot_ids,
-            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-        ).values_list("snapshot_id", "plugin", "hook_name"),
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
     )
-    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    missing_rows = rows_to_queue - existing_rows
     if missing_rows:
         ArchiveResult.objects.bulk_create(
             [
@@ -265,20 +272,49 @@ def run_plugins(
     queue_at = timezone.now()
     if existing_snapshot_ids:
         if requested_rows:
-            # Targeted ArchiveResult retries use retry_at as the scheduling
-            # signal and keep sealed snapshots sealed so extractors are not
-            # re-run outside the explicitly queued plugin rows. Paused snapshots
-            # also keep status=paused here: `retry_at` only asks the orchestrator
-            # to process the queued plugin rows, and run_due_snapshot restores
-            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "modified_at"):
+            # Search indexing on a sealed Snapshot is the only targeted hook
+            # allowed to bypass the normal lifecycle. Every other requested
+            # plugin requeues its Snapshot through the unified state machine.
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
+            for snapshot_id, plugin_name, _hook_name in requested_rows:
+                requested_plugins_by_id[snapshot_id].add(plugin_name)
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
-                snapshot.safe_update(
-                    {"retry_at": queue_at, "modified_at": queue_at},
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
+                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
+                sealed_search_backfill = (
+                    snapshot.status == Snapshot.StatusChoices.SEALED
+                    and plugin_names
+                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
                 )
+                if sealed_search_backfill:
+                    snapshot.safe_update(
+                        {"retry_at": queue_at, "modified_at": queue_at},
+                        refresh=False,
+                        extra_filter={"status": snapshot.status},
+                    )
+                else:
+                    snapshot.update_and_requeue(
+                        status=Snapshot.StatusChoices.QUEUED,
+                        retry_at=queue_at,
+                        current_step=0,
+                    )
         else:
             # No plugin rows were requested, so this is a full snapshot retry.
             for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):
@@ -378,16 +414,11 @@ def run_plugins(
 
 
 def is_archiveresult_id(value: str) -> bool:
-    """Check if value looks like an ArchiveResult UUID."""
-    import re
-
-    uuid_pattern = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
-    if not uuid_pattern.match(value):
-        return False
-    # Verify it's actually an ArchiveResult (not a Snapshot or other object)
+    """Check if value resolves to an ArchiveResult ID."""
     from archivebox.core.models import ArchiveResult
+    from archivebox.api.v1_core import _uuid_ref_query
 
-    return ArchiveResult.objects.filter(id=value).exists()
+    return ArchiveResult.objects.filter(_uuid_ref_query("id", value)).exists()
 
 
 @click.command()

archivebox/cli/archivebox_help.py

@@ -13,6 +13,10 @@ from rich.panel import Panel
 
 
 def _command_doc(cmd: str, import_path: str) -> str:
+    def first_doc_line(docstring: str | None) -> str:
+        lines = (docstring or "").splitlines()
+        return lines[0] if lines else ""
+
     modname, _ = import_path.rsplit(".", 1)
     spec = importlib.util.find_spec(modname)
     if spec is None or spec.origin is None:
@@ -26,9 +30,9 @@ def _command_doc(cmd: str, import_path: str) -> str:
     for name in (cmd, "main"):
         for node in tree.body:
             if isinstance(node, ast.FunctionDef) and node.name == name:
-                return (ast.get_docstring(node) or "").splitlines()[0]
+                return first_doc_line(ast.get_docstring(node))
 
-    return (ast.get_docstring(tree) or "").splitlines()[0]
+    return first_doc_line(ast.get_docstring(tree))
 
 
 def help() -> None:
@@ -85,8 +89,8 @@ def help() -> None:
     [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)
 
-    config = get_config()
-    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
+    get_config()
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
         EXAMPLE_USAGE = f"""
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

archivebox/cli/archivebox_init.py

@@ -25,20 +25,20 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.misc.checks import check_not_inside_source_dir
+
+    check_not_inside_source_dir()
+
+    from archivebox.config import CONSTANTS, VERSION
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
-    from archivebox.misc.db import apply_migrations
+    from archivebox.misc.db import apply_migrations, database_exists, ensure_database_ready
     from archivebox.misc.checks import check_migrations
 
     config = get_config()
 
-    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
-    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
-    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
-
-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
+    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    existing_index = database_exists()
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
         print("[green]----------------------------------------------------------------------[/green]")
@@ -65,46 +65,60 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     else:
         print("\n[green][+] Building archive folder structure...[/green]")
 
-    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
+    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
     print(f"    + {archive_path}, {sources_path}, {logs_path}...")
     Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-    for path in (Path(CONSTANTS.SOURCES_DIR), config.ARCHIVE_DIR, config.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
         path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
 
     # create the .archivebox_id file with a unique ID for this collection
     from archivebox.config.paths import _get_collection_id
 
-    _get_collection_id(DATA_DIR, force_create=True)
+    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
 
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
 
-    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
+    if existing_index:
         print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
     else:
         print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
 
+    # For postgres, make sure the server is reachable and create the database
+    # if it doesn't exist yet (sqlite creates its file automatically).
+    ensure_database_ready()
+
     from archivebox.config.django import setup_django
 
     setup_django()
-    check_migrations(blocking=True, auto_apply=False)
+    previous_wants_init = os.environ.get("ARCHIVEBOX_WANTS_INIT")
+    os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
+    try:
+        check_migrations(blocking=True, auto_apply=False)
 
-    for migration_line in apply_migrations(DATA_DIR):
-        sys.stdout.write(f"    {migration_line}\n")
+        for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
+            sys.stdout.write(f"    {migration_line}\n")
+    finally:
+        if previous_wants_init is None:
+            os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
+        else:
+            os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
-    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
+    from archivebox.misc.db import database_display_location, is_postgres
+
+    assert database_exists()
     print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
-
-    # from django.contrib.auth.models import User
-    #     call_command("createsuperuser", interactive=True)
+    if is_postgres():
+        print(f"    √ {database_display_location()}")
+    else:
+        print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
 
     print()
     print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
@@ -141,12 +155,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
     CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
 
-    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
+    from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
 
     config = get_config()
-    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+    create_and_chown_dir(config.TMP_DIR)
+    create_and_chown_dir(config.ABXPKG_LIB_DIR)
+    create_and_chown_dir(config.ABXPKG_LIB_DIR / "bin")
 
     working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
     if working_tmp_dir:
@@ -155,7 +169,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method
@@ -179,7 +192,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 @click.command()
 @click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
 @click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
 @docstring(init.__doc__)
 def main(**kwargs) -> None:
     init(**kwargs)

archivebox/cli/archivebox_install.py

@@ -3,6 +3,9 @@
 __package__ = "archivebox.cli"
 
 import os
+import shutil
+import re
+from pathlib import Path
 
 import rich_click as click
 from rich import print
@@ -10,6 +13,108 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types
 
 
+def _resolve_install_targets(
+    requested_names: tuple[str, ...],
+) -> tuple[list[str], list[str]]:
+    """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
+    from abx_dl.models import discover_plugins, filter_plugins
+
+    plugins = discover_plugins(runtime="archivebox")
+    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in plugins}
+    plugin_names_by_binary_alias: dict[str, set[str]] = {}
+    for plugin_name, plugin in plugins.items():
+        for required_binary in plugin.config.required_binaries:
+            aliases = {required_binary.name.lower()}
+            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
+            if template_match:
+                config_key = template_match.group(1)
+                aliases.add(config_key.lower())
+                aliases.add(config_key.removesuffix("_BINARY").lower())
+                default = plugin.config.properties.get(config_key, {}).get("default")
+                if default:
+                    aliases.add(str(default).lower())
+            for alias in aliases:
+                plugin_names_by_binary_alias.setdefault(alias, set()).add(plugin_name)
+
+    requested_plugins: list[str] = []
+    raw_binary_names: list[str] = []
+    for name in requested_names:
+        normalized_name = name.lower()
+        if normalized_name in plugin_names_by_lower:
+            requested_plugins.append(plugin_names_by_lower[normalized_name])
+        elif normalized_name in plugin_names_by_binary_alias:
+            requested_plugins.extend(sorted(plugin_names_by_binary_alias[normalized_name]))
+        else:
+            raw_binary_names.append(name)
+
+    selected_plugins = filter_plugins(plugins, requested_plugins, include_providers=True) if requested_plugins else {}
+    selected_plugin_names = {name.lower() for name in selected_plugins}
+    raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
+    return sorted(selected_plugins), sorted(set(raw_binary_names))
+
+
+def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
+    """Install user-requested standalone binaries through the Binary state machine."""
+    from django.utils import timezone
+
+    from archivebox.machine.models import Binary, Machine, _canonical_binary_name
+    from archivebox.services.runner import run_due_binary
+
+    machine = Machine.current()
+    for requested_name in binary_names:
+        binary_name = _canonical_binary_name(requested_name)
+        if not binary_name:
+            raise click.UsageError(f"Invalid binary name: {requested_name}")
+        binary = Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").first()
+        if binary is None:
+            binary = Binary.objects.create(
+                machine=machine,
+                name=binary_name,
+                binproviders=binproviders,
+                overrides={},
+                status=Binary.StatusChoices.QUEUED,
+            )
+        elif not binary.is_valid or binary.binproviders != binproviders:
+            binary.binproviders = binproviders
+            binary.overrides = {}
+            binary.status = Binary.StatusChoices.QUEUED
+            binary.retry_at = timezone.now()
+            binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
+        run_due_binary(binary, lock_seconds=60)
+
+
+def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
+    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
+    link_path = data_dir / "lib"
+    target_config_path = abxpkg_lib_dir.expanduser().absolute()
+    target_path = abxpkg_lib_dir.expanduser().resolve()
+    from archivebox.config.paths import get_machine_type
+
+    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
+
+    if target_config_path == scoped_data_lib_path:
+        if link_path.is_symlink() or link_path.is_file():
+            link_path.unlink()
+        target_config_path.mkdir(parents=True, exist_ok=True)
+        return None
+
+    if link_path.resolve(strict=False) == target_path:
+        return None
+
+    target_path.mkdir(parents=True, exist_ok=True)
+
+    if link_path.is_symlink():
+        if link_path.resolve(strict=False) == target_path:
+            return link_path
+        link_path.unlink()
+    elif link_path.exists():
+        shutil.rmtree(link_path)
+
+    relative_target = os.path.relpath(target_path, start=link_path.parent)
+    link_path.symlink_to(relative_target, target_is_directory=True)
+    return link_path
+
+
 @enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
     """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -22,12 +127,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     """
 
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.misc.logging import stderr
     from archivebox.cli.archivebox_init import init
 
-    config = get_config()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
 
     if dry_run:
         print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")
@@ -57,16 +161,32 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     setup_django()
 
-    plugin_names = list(binaries)
+    plugin_names, raw_binary_names = _resolve_install_targets(binaries) if binaries else ([], [])
+    install_plugin_names = list(plugin_names)
     if binproviders != "*":
-        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
+        install_plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
 
-    print("[+] Running installer via abx-dl bus...")
-    print()
+    if install_plugin_names or not binaries:
+        if not binaries:
+            from archivebox.workers.supervisord_util import require_dependency_install_memory
 
-    from archivebox.services.runner import run_install
+            require_dependency_install_memory()
+        print("[+] Running plugin installer via abx-dl bus...")
+        print()
 
-    run_install(plugin_names=plugin_names or None)
+        from archivebox.services.runner import run_install
+
+        run_install(plugin_names=install_plugin_names or None)
+
+    if raw_binary_names:
+        print(f"[+] Running direct binary installer via ArchiveBox binary state machine: {', '.join(raw_binary_names)}")
+        print()
+        _install_raw_binary_names(raw_binary_names, binproviders)
+
+    from archivebox.config.common import get_config
+
+    config = get_config(include_machine=False)
+    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
 
     print()
 
@@ -85,7 +205,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     # Django is already loaded, so just import and call the function directly
     from archivebox.cli.archivebox_version import version as show_version
 
-    show_version(quiet=False)
+    show_version(quiet=False, binaries=binaries)
 
 
 @click.command()

archivebox/cli/archivebox_list.py

@@ -7,50 +7,15 @@ import sys
 
 import rich_click as click
 
-from archivebox.cli.archivebox_snapshot import list_snapshots
+from archivebox.cli.archivebox_snapshot import list_snapshots, snapshot_filter_options, snapshot_output_options
 
 
 @click.command()
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def main(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-) -> None:
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def main(**kwargs) -> None:
     """List Snapshots."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_manage.py

@@ -14,7 +14,7 @@ def manage(args: list[str] | None = None) -> None:
     from archivebox.misc.logging import stderr
 
     config = get_config()
-    if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY):
+    if (args and "createsuperuser" in args and "--noinput" not in args) and (config.IN_DOCKER and not config.IS_TTY):
         stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
         stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
         stderr("")

archivebox/cli/archivebox_oneshot.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox oneshot"
+
+import subprocess
+import sys
+from pathlib import Path
+
+import rich_click as click
+
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
+def main(args: tuple[str, ...] = ()) -> None:
+    """Download URLs using abx-dl"""
+    from archivebox.misc.checks import check_not_inside_source_dir
+
+    check_not_inside_source_dir()
+
+    cwd = Path.cwd()
+    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
+        raise click.ClickException(
+            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
+        )
+    abxpkg_binary = Path(sys.executable).with_name("abxpkg")
+    if not abxpkg_binary.is_file():
+        raise click.ClickException(f"abxpkg executable is missing from the ArchiveBox environment: {abxpkg_binary}")
+    abxpkg_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
+    raise SystemExit(
+        subprocess.run(
+            [
+                str(abxpkg_binary),
+                f"--lib={abxpkg_lib_dir}",
+                "--binproviders=env",
+                "--install",
+                "run",
+                "abx-dl",
+                *args,
+            ],
+        ).returncode,
+    )
+
+
+if __name__ == "__main__":
+    main()

archivebox/cli/archivebox_persona.py

@@ -137,65 +137,6 @@ def get_edge_user_data_dir() -> Path | None:
     return None
 
 
-def get_browser_binary(browser: str) -> str | None:
-    system = platform.system()
-    home = Path.home()
-    browser = browser.lower()
-
-    if system == "Darwin":
-        candidates = {
-            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
-            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
-            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
-            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
-        }.get(browser, [])
-    elif system == "Linux":
-        candidates = {
-            "chrome": [
-                "/usr/bin/google-chrome",
-                "/usr/bin/google-chrome-stable",
-                "/usr/bin/google-chrome-beta",
-                "/usr/bin/google-chrome-unstable",
-            ],
-            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
-            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
-            "edge": [
-                "/usr/bin/microsoft-edge",
-                "/usr/bin/microsoft-edge-stable",
-                "/usr/bin/microsoft-edge-beta",
-                "/usr/bin/microsoft-edge-dev",
-            ],
-        }.get(browser, [])
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = {
-            "chrome": [
-                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
-                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
-                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
-            ],
-            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
-            "brave": [
-                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
-                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-            ],
-            "edge": [
-                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
-                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
-                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
-            ],
-        }.get(browser, [])
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate and Path(candidate).exists():
-            return candidate
-
-    return None
-
-
 BROWSER_PROFILE_FINDERS = {
     "chrome": get_chrome_user_data_dir,
     "chromium": get_chrome_user_data_dir,  # Same locations
@@ -317,10 +258,6 @@ def create_personas(
         if profile is None and (source_profile_dir / "Default").exists():
             profile = "Default"
 
-        browser_binary = get_browser_binary(import_from)
-        if browser_binary:
-            rprint(f"[dim]Using {import_from} binary: {browser_binary}[/dim]", file=sys.stderr)
-
     created_count = 0
     for name in name_list:
         name = name.strip()
@@ -372,7 +309,7 @@ def create_personas(
         if not is_tty:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -424,7 +361,7 @@ def list_personas(
         else:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -500,7 +437,7 @@ def update_personas(name: str | None = None) -> int:
             if not is_tty:
                 write_record(
                     {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "id": str(persona.id),
                         "name": persona.name,
                         "path": str(persona.path),
                     },

archivebox/cli/archivebox_remove.py

@@ -9,10 +9,9 @@ from collections.abc import Iterable
 
 import rich_click as click
 
-from django.db import OperationalError
 from django.db.models import QuerySet
 
-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -23,6 +22,7 @@ from archivebox.misc.logging_util import (
     log_removal_finished,
     TimedProgress,
 )
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 
 @enforce_types
@@ -33,49 +33,56 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet:
+    delete: bool = False,
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    timeout: float | None = None,
+    **kwargs,
+) -> dict[str, object]:
     """Remove the specified URLs from the archive"""
 
     setup_django()
     check_data_folder()
+    timeout = float(timeout) if timeout is not None else None
 
-    from archivebox.cli.archivebox_search import get_snapshots
+    from archivebox.core.models import Snapshot
 
+    filter_kwargs = {
+        **kwargs,
+        "filter_patterns": filter_patterns,
+        "filter_type": filter_type,
+        "after": after,
+        "before": before,
+    }
     pattern_list = list(filter_patterns)
 
     log_list_started(pattern_list or None, filter_type)
     timer = TimedProgress(360, prefix="      ")
     try:
-        snapshots = get_snapshots(
-            snapshots=snapshots,
-            filter_patterns=pattern_list or None,
-            filter_type=filter_type,
-            after=after,
-            before=before,
-        )
+        if snapshots is None:
+            snapshots = Snapshot.objects.order_by("-created_at").search(**filter_kwargs)
+        # Freeze the target set up-front so a concurrent daemon writing new
+        # snapshots can't extend the deletion under us, and so the cursor isn't
+        # held open across the per-row deletes below.
+        snapshot_pks = list(snapshots.values_list("pk", flat=True))
     finally:
         timer.end()
 
-    if not snapshots.exists():
+    if not snapshot_pks:
         log_removal_finished(0, 0)
         raise SystemExit(1)
 
-    log_list_finished(snapshots)
-    log_removal_started(snapshots, yes=yes)
+    if not yes:
+        log_list_finished(snapshots)
+        log_removal_started(snapshots, yes=False)
 
-    from archivebox.core.models import Snapshot
     from archivebox.search.query import flush_search_index
 
-    # Freeze the target set up-front so a concurrent daemon writing new
-    # snapshots can't extend the deletion under us, and so the cursor isn't
-    # held open across the per-row deletes below.
-    snapshot_pks = list(snapshots.values_list("pk", flat=True))
-    to_remove = len(snapshot_pks)
+    started_at = time.monotonic()
+    deadline = started_at + timeout if timeout is not None else None
 
     # Search-index flush touches a separate backend (FTS / sonic), not the
     # main index.sqlite3 writer lock, so it's safe to do once up front.
-    flush_search_index(snapshots=Snapshot.objects.filter(pk__in=snapshot_pks))
+    flush_search_index(snapshots=snapshots)
 
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
@@ -84,46 +91,46 @@ def remove(
     # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
     # rmtree doesn't hold the lock either.
     #
-    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
-    # errors when called inside an atomic block (because it can't safely
-    # release+reacquire a transaction), so we wrap each row's delete in our
-    # own retry loop at this outer (non-atomic) level. Each attempt is a
-    # fresh atomic; an exception cleanly rolls it back before we sleep.
-    retry_interval = 1.0
-    retry_timeout = 60.0
-    for pk in snapshot_pks:
-        deadline = time.monotonic() + retry_timeout
-        while True:
-            try:
-                Snapshot.objects.filter(pk=pk).delete()
-                break
-            except OperationalError as err:
-                if "database is locked" not in str(err) or time.monotonic() >= deadline:
-                    raise
-                time.sleep(retry_interval)
+    deleted_snapshot_pks = []
+    timed_out = False
+    timeout_error = ""
+    for index, pk in enumerate(snapshot_pks):
+        if deadline is not None and time.monotonic() >= deadline:
+            timed_out = True
+            timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
+            break
+        deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+        if deleted_count:
+            deleted_snapshot_pks.append(pk)
 
     all_snapshots = Snapshot.objects.all()
-    log_removal_finished(all_snapshots.count(), to_remove)
+    remaining_count = all_snapshots.count()
+    deleted_snapshot_id_set = set(deleted_snapshot_pks)
+    remaining_snapshot_pks = [snapshot_id for snapshot_id in snapshot_pks if snapshot_id not in deleted_snapshot_id_set]
+    log_removal_finished(remaining_count, len(deleted_snapshot_pks))
 
-    return all_snapshots
+    return {
+        "removed_count": len(deleted_snapshot_pks),
+        "removed_snapshot_ids": [str(snapshot_id) for snapshot_id in deleted_snapshot_pks],
+        "not_removed_count": len(remaining_snapshot_pks),
+        "not_removed_snapshot_ids": [str(snapshot_id) for snapshot_id in remaining_snapshot_pks],
+        "success": not timed_out,
+        "error": timeout_error,
+        "timeout": timeout,
+    }
 
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
-@click.option("--before", type=float, help="Remove only URLs bookmarked before timestamp")
-@click.option("--after", type=float, help="Remove only URLs bookmarked after timestamp")
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(("exact", "substring", "domain", "regex", "tag")),
-    default="exact",
-    help="Type of pattern matching to use when filtering URLs",
-)
-@click.argument("filter_patterns", nargs=-1)
+@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
+@click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)
 def main(**kwargs):
     """Remove the specified URLs from the archive"""
-    remove(**kwargs)
+    result = remove(**kwargs)
+    if not result["success"]:
+        raise SystemExit(124)
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_run.py

@@ -40,14 +40,23 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
-import sys
 import asyncio
+import os
+import signal
+import sys
 from collections import defaultdict
 
 import rich_click as click
 from rich import print as rprint
 
 
+RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
+
+
+def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    os._exit(128 + int(sig))
+
+
 def process_stdin_records() -> int:
     """
     Process JSONL records from stdin.
@@ -234,11 +243,17 @@ def process_stdin_records() -> int:
                     crawl_id,
                     snapshot_ids=None if crawl_id in full_crawl_ids else sorted(snapshot_ids_by_crawl[crawl_id]),
                     selected_plugins=None if crawl_id in run_all_plugins_for_crawl else sorted(plugin_names_by_crawl[crawl_id]),
+                    selected_plugins_are_explicit=False,
                 )
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
     """
     Run the background runner loop.
 
@@ -250,47 +265,68 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
     from archivebox.config import CONSTANTS
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
     from archivebox.machine.models import Machine, Process
-    from archivebox.core.takeover_util import enter_single_runner_gate
-    from archivebox.services.runner import recover_orchestrator_state, run_pending_crawls
+    from archivebox.core.takeover_util import enter_single_runner_gate, standby_until_foreground_runner_needed
+    from archivebox.core.recovery_util import recover_orchestrator_state
+    from archivebox.services.runner import run_pending_crawls
 
     Machine.current()
     current = Process.current()
+    root_command = current.root
+    if daemon and root_command.process_type in (
+        Process.TypeChoices.SERVER,
+        Process.TypeChoices.ADD,
+        Process.TypeChoices.UPDATE,
+    ):
+        # Server-owned daemon runners are persistent supervisor workers, but
+        # foreground add/update commands are allowed to borrow runner/sonic
+        # leadership without taking down Daphne. Waiting here keeps the worker
+        # on the normal runner path while preventing a server restart loop from
+        # immediately stealing the single-runner gate back from the newer CLI.
+        standby_until_foreground_runner_needed(root_command, data_dir=CONSTANTS.DATA_DIR)
     if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
         current.mark_exited()
         return 0
 
-    recover_orchestrator_state(include_chrome=True)
+    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
     if crawl_id:
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
 
         crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
         now = timezone.now()
-        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
-        # existing lease has already expired. A future retry_at means another
-        # worker is already scheduled — don't clobber.
-        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
-            # extra_filter pins the read-time retry_at so a concurrent worker
-            # that grabbed the lease between our SELECT and UPDATE wins.
-            crawl.safe_update(
-                {"retry_at": now},
-                refresh=False,
-                extra_filter={"retry_at": crawl.retry_at},
-            )
+        # Winning the single-runner gate terminates and waits for every older
+        # runner. Requeue the explicitly requested crawl so an abandoned
+        # future ownership lease cannot hide it from the unified scheduler.
+        if crawl is not None:
+            crawl.update_and_requeue(retry_at=now, refresh=False)
     # Only a foreground `archivebox add` gets the interactive "abort current
     # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
     # orchestrators should shut down immediately and cleanly on the first signal.
     interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
             run_pending_crawls(
                 daemon=daemon,
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
-    except (KeyboardInterrupt, asyncio.CancelledError):
+    except KeyboardInterrupt:
+        return 0
+    except asyncio.CancelledError as e:
+        if daemon:
+            rprint(f"[red]Runner cancelled unexpectedly: {type(e).__name__}: {e}[/red]", file=sys.stderr)
+            return 1
         return 0
     except Exception as e:
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
@@ -306,8 +342,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool):
+@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
+@click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
     """
     Process queued work.
 
@@ -322,8 +374,15 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
 
     if daemon and not snapshot_id and not binary_id and not crawl_id:
         try:
-            with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=False):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
+            os.environ[RUNNER_DAEMON_ENV] = "1"
+            with (
+                foreground_shutdown_signals(
+                    on_signal=_exit_daemon_runner_on_signal,
+                    raise_on_first_signal=False,
+                ),
+                foreground_parent_watchdog(enabled=False),
+            ):
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
         except KeyboardInterrupt:
             sys.exit(0)
 
@@ -347,30 +406,49 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 sys.exit(1)
 
         if crawl_id:
-            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
 
-        if not sys.stdin.isatty():
+        if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_due_snapshot
     from django.utils import timezone
 
+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
     snapshot = None
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
-            snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
-            if snapshot.retry_at is None:
-                snapshot.update_and_requeue(retry_at=timezone.now())
-            run_due_snapshot(snapshot, lock_seconds=60)
+            for _ in range(10):
+                snapshot = Snapshot.objects.select_related("crawl").get(id=snapshot_id)
+                if snapshot.retry_at is None:
+                    snapshot.update_and_requeue(retry_at=timezone.now())
+                elif snapshot.retry_at > timezone.now():
+                    break
+                if not run_due_snapshot(snapshot, lock_seconds=60):
+                    break
         return 0
     except KeyboardInterrupt:
         try:
@@ -389,6 +467,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 
         traceback.print_exc()
         return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_search.py

@@ -3,255 +3,7 @@
 __package__ = "archivebox.cli"
 __command__ = "archivebox search"
 
-import sys
-from pathlib import Path
-from typing import TYPE_CHECKING
-from collections.abc import Callable
-
-import rich_click as click
-
-from django.db.models import Q, QuerySet
-
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
-from archivebox.misc.logging import stderr
-from archivebox.misc.util import enforce_types, docstring
-
-if TYPE_CHECKING:
-    from archivebox.core.models import Snapshot
-
-# Filter types for URL matching
-LINK_FILTERS: dict[str, Callable[[str], Q]] = {
-    "exact": lambda pattern: Q(url=pattern),
-    "substring": lambda pattern: Q(url__icontains=pattern),
-    "regex": lambda pattern: Q(url__iregex=pattern),
-    "domain": lambda pattern: (
-        Q(url__istartswith=f"http://{pattern}") | Q(url__istartswith=f"https://{pattern}") | Q(url__istartswith=f"ftp://{pattern}")
-    ),
-    "tag": lambda pattern: Q(tags__name=pattern),
-    "timestamp": lambda pattern: Q(timestamp=pattern),
-}
-
-STATUS_CHOICES = ["indexed", "archived", "unarchived"]
-
-
-def _apply_pattern_filters(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    filter_patterns: list[str],
-    filter_type: str,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    filter_builder = LINK_FILTERS.get(filter_type)
-    if filter_builder is None:
-        stderr()
-        stderr(f"[X] Got invalid pattern for --filter-type={filter_type}", color="red")
-        raise SystemExit(2)
-
-    query = Q()
-    for pattern in filter_patterns:
-        query |= filter_builder(pattern)
-    return snapshots.filter(query)
-
-
-def _snapshots_to_json(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from archivebox.config import VERSION
-    from archivebox.misc.util import to_json
-
-    config = get_config()
-    main_index_header = (
-        {
-            "info": "This is an index of site data archived by ArchiveBox: The self-hosted web archive.",
-            "schema": "archivebox.index.json",
-            "copyright_info": config.FOOTER_INFO,
-            "meta": {
-                "project": "ArchiveBox",
-                "version": VERSION,
-                "git_sha": VERSION,
-                "website": "https://ArchiveBox.io",
-                "docs": "https://github.com/ArchiveBox/ArchiveBox/wiki",
-                "source": "https://github.com/ArchiveBox/ArchiveBox",
-                "issues": "https://github.com/ArchiveBox/ArchiveBox/issues",
-                "dependencies": {},
-            },
-        }
-        if with_headers
-        else {}
-    )
-
-    snapshot_dicts = [snapshot.to_dict(extended=True) for snapshot in snapshots.iterator(chunk_size=500)]
-    output: dict[str, object] | list[dict[str, object]]
-    if with_headers:
-        output = {
-            **main_index_header,
-            "num_links": len(snapshot_dicts),
-            "updated": datetime.now(tz.utc),
-            "last_run_cmd": sys.argv,
-            "links": snapshot_dicts,
-        }
-    else:
-        output = snapshot_dicts
-
-    return to_json(output, indent=4, sort_keys=True)
-
-
-def _snapshots_to_csv(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    cols: list[str],
-    with_headers: bool,
-) -> str:
-    header = ",".join(cols) if with_headers else ""
-    rows = [snapshot.to_csv(cols=cols, separator=",") for snapshot in snapshots.iterator(chunk_size=500)]
-    return "\n".join((header, *rows))
-
-
-def _snapshots_to_html(
-    snapshots: QuerySet["Snapshot", "Snapshot"],
-    *,
-    with_headers: bool,
-) -> str:
-    from datetime import datetime, timezone as tz
-
-    from django.template.loader import render_to_string
-
-    from archivebox.config import VERSION
-    from archivebox.config.version import get_COMMIT_HASH
-
-    config = get_config()
-    template = "static_index.html" if with_headers else "minimal_index.html"
-    snapshot_list = list(snapshots.iterator(chunk_size=500))
-
-    return render_to_string(
-        template,
-        {
-            "version": VERSION,
-            "git_sha": get_COMMIT_HASH() or VERSION,
-            "num_links": str(len(snapshot_list)),
-            "date_updated": datetime.now(tz.utc).strftime("%Y-%m-%d"),
-            "time_updated": datetime.now(tz.utc).strftime("%Y-%m-%d %H:%M"),
-            "links": snapshot_list,
-            "FOOTER_INFO": config.FOOTER_INFO,
-        },
-    )
-
-
-def get_snapshots(
-    snapshots: QuerySet["Snapshot", "Snapshot"] | None = None,
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    after: float | None = None,
-    before: float | None = None,
-    out_dir: Path = DATA_DIR,
-) -> QuerySet["Snapshot", "Snapshot"]:
-    """Filter and return Snapshots matching the given criteria."""
-    from archivebox.core.models import Snapshot
-
-    if snapshots is not None:
-        result = snapshots
-    else:
-        result = Snapshot.objects.all()
-
-    if after is not None:
-        result = result.filter(timestamp__gte=after)
-    if before is not None:
-        result = result.filter(timestamp__lt=before)
-    if filter_patterns:
-        result = _apply_pattern_filters(result, filter_patterns, filter_type)
-
-    # Prefetch crawl relationship to avoid N+1 queries when accessing output_dir
-    result = result.select_related("crawl", "crawl__created_by")
-
-    if not result.exists():
-        stderr("[!] No Snapshots matched your filters:", filter_patterns, f"({filter_type})", color="lightyellow")
-
-    return result
-
-
-@enforce_types
-def search(
-    filter_patterns: list[str] | None = None,
-    filter_type: str = "substring",
-    status: str = "indexed",
-    before: float | None = None,
-    after: float | None = None,
-    sort: str | None = None,
-    json: bool = False,
-    html: bool = False,
-    csv: str | None = None,
-    with_headers: bool = False,
-):
-    """List, filter, and export information about archive entries"""
-
-    if with_headers and not (json or html or csv):
-        stderr("[X] --with-headers requires --json, --html or --csv\n", color="red")
-        raise SystemExit(2)
-
-    # Query DB directly - no filesystem scanning
-    snapshots = get_snapshots(
-        filter_patterns=list(filter_patterns) if filter_patterns else None,
-        filter_type=filter_type,
-        before=before,
-        after=after,
-    )
-
-    # Apply status filter
-    if status == "archived":
-        snapshots = snapshots.filter(downloaded_at__isnull=False)
-    elif status == "unarchived":
-        snapshots = snapshots.filter(downloaded_at__isnull=True)
-    # 'indexed' = all snapshots (no filter)
-
-    if sort:
-        snapshots = snapshots.order_by(sort)
-
-    # Export to requested format
-    if json:
-        output = _snapshots_to_json(snapshots, with_headers=with_headers)
-    elif html:
-        output = _snapshots_to_html(snapshots, with_headers=with_headers)
-    elif csv:
-        output = _snapshots_to_csv(snapshots, cols=csv.split(","), with_headers=with_headers)
-    else:
-        from archivebox.misc.logging_util import printable_folders
-
-        # Convert to dict for printable_folders
-        folders: dict[str, Snapshot | None] = {str(snapshot.output_dir): snapshot for snapshot in snapshots}
-        output = printable_folders(folders, with_headers)
-
-    # Structured exports must be written directly to stdout.
-    # rich.print() reflows long lines to console width, which corrupts JSON/CSV/HTML output.
-    sys.stdout.write(output)
-    if not output.endswith("\n"):
-        sys.stdout.write("\n")
-    return output
-
-
-@click.command()
-@click.option(
-    "--filter-type",
-    "-f",
-    type=click.Choice(["search", *LINK_FILTERS.keys()]),
-    default="substring",
-    help="Pattern matching type for filtering URLs",
-)
-@click.option("--status", "-s", type=click.Choice(STATUS_CHOICES), default="indexed", help="List snapshots with the given status")
-@click.option("--before", "-b", type=float, help="List snapshots bookmarked before the given UNIX timestamp")
-@click.option("--after", "-a", type=float, help="List snapshots bookmarked after the given UNIX timestamp")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--json", "-J", is_flag=True, help="Print output in JSON format")
-@click.option("--html", "-M", is_flag=True, help="Print output in HTML format (suitable for viewing statically without a server)")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: created_at,url,title")
-@click.option("--with-headers", "-H", is_flag=True, help="Include extra CSV/HTML headers in the output")
-@click.help_option("--help", "-h")
-@click.argument("filter_patterns", nargs=-1)
-@docstring(search.__doc__)
-def main(**kwargs):
-    return search(**kwargs)
+from archivebox.cli.archivebox_list import main
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -12,6 +12,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
 
 
@@ -230,6 +231,7 @@ def server(
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
+    createsuperuser: bool = False,
 ) -> None:
     """Run the ArchiveBox HTTP server"""
     from archivebox.config.common import get_config
@@ -243,6 +245,12 @@ def server(
 
     from django.contrib.auth.models import User
 
+    if createsuperuser:
+        from archivebox.cli.archivebox_manage import manage
+
+        manage(args=["createsuperuser"])
+        print()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         print()
         print(
@@ -259,13 +267,13 @@ def server(
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
     if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
-        from archivebox.config import CONSTANTS
+        from archivebox.workers.supervisord_util import resolve_env_binary
 
         log_path = CONSTANTS.LOGS_DIR / "server.log"
         log_path.parent.mkdir(parents=True, exist_ok=True)
         daemon_env = os.environ.copy()
         daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1"
-        daemon_cmd = [sys.executable, "-m", "archivebox", "server"]
+        daemon_cmd = [str(resolve_env_binary("archivebox")), "server"]
         if debug:
             daemon_cmd.append("--debug")
         if reload:
@@ -294,7 +302,14 @@ def server(
             except OSError:
                 time.sleep(0.1)
         else:
-            print(f"[yellow][!] ArchiveBox daemon server pid={proc.pid} is still starting. See {log_path}[/yellow]")
+            print(f"[red][X] ArchiveBox daemon server pid={proc.pid} did not become ready. See {log_path}[/red]")
+            proc.terminate()
+            try:
+                proc.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                proc.wait(timeout=5)
+            sys.exit(1)
         return
 
     os.environ["BIND_ADDR"] = f"{host}:{port}"
@@ -314,6 +329,7 @@ def server(
     from archivebox.core.takeover_util import (
         command_owns_runtime_stack,
         current_command,
+        foreground_runner_owner,
         runtime_stack_owner,
         standby_until_runtime_stack_needed,
     )
@@ -337,22 +353,29 @@ def server(
     runtime_config = get_config()
     _print_server_startup_warnings(runtime_config, host, port)
     bind_url = f"http://{host}:{port}"
-    command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
+    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
 
     def still_owns_runtime_stack() -> bool:
         from django.db import connections
 
         try:
-            return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
         finally:
             connections.close_all()
 
+    shutdown_state = None
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"):
+        with (
+            foreground_shutdown_signals() as shutdown_state,
+            foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
+        ):
             while True:
-                standby_result = standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=config.DATA_DIR, exclude_id=command.id)
-                takeover_components = active_supervisord_runtime_components(config=config)
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id) or foreground_runner_owner(
+                    data_dir=CONSTANTS.DATA_DIR,
+                    exclude_id=command.id,
+                )
+                takeover_components = active_supervisord_runtime_components()
                 if older_owner and takeover_components:
                     print(
                         "[yellow][*] Taking over "
@@ -386,7 +409,8 @@ def server(
     except KeyboardInterrupt:
         pass
     finally:
-        command.mark_exited()
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 
@@ -396,6 +420,7 @@ def server(
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
+@click.option("--createsuperuser", is_flag=True, help="Run archivebox manage createsuperuser before starting the server")
 @docstring(server.__doc__)
 def main(**kwargs):
     server(**kwargs)

archivebox/cli/archivebox_shell.py

@@ -12,9 +12,10 @@ from archivebox.misc.util import docstring
 def shell(args: Iterable[str] = ()) -> None:
     """Enter an interactive ArchiveBox Django shell"""
 
-    from django.core.management import call_command
+    from django.core.management import call_command, get_commands
 
-    call_command("shell_plus", *args)
+    shell_command = "shell_plus" if "shell_plus" in get_commands() else "shell"
+    call_command(shell_command, *args)
 
 
 @click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))

archivebox/cli/archivebox_snapshot.py

@@ -31,13 +31,62 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox snapshot"
 
 import sys
-from collections.abc import Iterable
+from collections.abc import Iterable, Iterator
+from itertools import islice
 
 import rich_click as click
 from rich import print as rprint
-from django.db.models import Case, IntegerField, QuerySet, When
+from django.db.models import QuerySet
 
-from archivebox.cli.cli_util import apply_filters
+SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
+SNAPSHOT_LIST_CHUNK_SIZE = 5000
+
+
+def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
+    from archivebox.config import VERSION
+    from archivebox.core.models import SnapshotTag
+
+    fields = (
+        "id",
+        "crawl_id",
+        "url",
+        "title",
+        "bookmarked_at",
+        "created_at",
+        "timestamp",
+        "depth",
+        "status",
+        "fs_version",
+        "output_size",
+    )
+    rows = queryset.values(*fields).iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE)
+    while batch := list(islice(rows, SNAPSHOT_LIST_CHUNK_SIZE)):
+        tags_by_snapshot = {row["id"]: [] for row in batch}
+        tag_rows = (
+            SnapshotTag.objects.filter(snapshot_id__in=tags_by_snapshot).order_by("tag__name").values_list("snapshot_id", "tag__name")
+        )
+        for snapshot_id, tag_name in tag_rows:
+            tags_by_snapshot[snapshot_id].append(tag_name)
+
+        for row in batch:
+            archive_size = int(row["output_size"] or 0)
+            yield {
+                "type": "Snapshot",
+                "schema_version": VERSION,
+                "id": str(row["id"]),
+                "crawl_id": str(row["crawl_id"]),
+                "url": row["url"],
+                "title": row["title"],
+                "tags": ",".join(sorted(tags_by_snapshot[row["id"]])),
+                "bookmarked_at": row["bookmarked_at"].isoformat() if row["bookmarked_at"] else None,
+                "created_at": row["created_at"].isoformat() if row["created_at"] else None,
+                "timestamp": row["timestamp"],
+                "depth": row["depth"],
+                "status": row["status"],
+                "fs_version": row["fs_version"],
+                "archive_size": archive_size,
+                "output_size": archive_size,
+            }
 
 
 # =============================================================================
@@ -66,7 +115,7 @@ def create_snapshots(
         TYPE_SNAPSHOT,
         TYPE_CRAWL,
     )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
@@ -111,7 +160,7 @@ def create_snapshots(
                 # Create snapshots for each URL in the crawl
                 for url in crawl.get_urls_list():
                     try:
-                        validate_url_length(url)
+                        validate_url(url)
                     except ValueError as err:
                         rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                         continue
@@ -134,7 +183,7 @@ def create_snapshots(
             elif record_type == TYPE_SNAPSHOT or record.get("url"):
                 # Input is a Snapshot or plain URL
                 if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                 if tag and not record.get("tags"):
                     record["tags"] = tag
                 if status:
@@ -178,72 +227,63 @@ def create_snapshots(
 # =============================================================================
 
 
+def snapshot_filter_options(*, default_filter_type: str):
+    def decorate(func):
+        for decorator in reversed(
+            (
+                click.option("--status", "-s", help="Filter by status (queued, started, sealed)"),
+                click.option("--url__icontains", help="Filter by URL contains"),
+                click.option("--url__istartswith", help="Filter by URL starts with"),
+                click.option("--tag", "-t", help="Filter by tag name"),
+                click.option("--crawl-id", help="Filter by crawl ID"),
+                click.option("--limit", "-n", type=int, help="Limit number of results"),
+                click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at"),
+                click.option("--search", help="Search mode to use for positional query"),
+                click.option("--before", type=float, help="Only snapshots bookmarked before timestamp"),
+                click.option("--after", type=float, help="Only snapshots bookmarked after timestamp"),
+                click.option(
+                    "--filter-type",
+                    "-f",
+                    type=click.Choice(SNAPSHOT_FILTER_TYPE_CHOICES),
+                    default=default_filter_type,
+                    help="Type of pattern matching to use for positional filters",
+                ),
+                click.argument("filter_patterns", nargs=-1),
+            ),
+        ):
+            func = decorator(func)
+        return func
+
+    return decorate
+
+
+def snapshot_output_options(func):
+    for decorator in reversed(
+        (
+            click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title"),
+            click.option("--json", "as_json", is_flag=True, help="Print output as a JSON array"),
+            click.option("--html", "as_html", is_flag=True, help="Print output as HTML"),
+            click.option("--with-headers", is_flag=True, help="Include column headers in structured output"),
+        ),
+    ):
+        func = decorator(func)
+    return func
+
+
 def build_snapshot_queryset(
-    *,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    query: str | None = None,
-    limit: int | None = None,
+    **kwargs,
 ) -> QuerySet:
     from archivebox.core.models import Snapshot
-    from archivebox.search.query import apply_snapshot_search
 
-    queryset = Snapshot.objects.order_by("-created_at")
-    queryset = apply_filters(
-        queryset,
-        {
-            "status": status,
-            "url__icontains": url__icontains,
-            "url__istartswith": url__istartswith,
-            "crawl_id": crawl_id,
-        },
-    )
-
-    if tag:
-        queryset = queryset.filter(tags__name__iexact=tag)
-
-    query = (query or "").strip()
-    if query:
-        try:
-            queryset = apply_snapshot_search(
-                queryset,
-                query,
-                search_mode=search,
-                ordering=("-created_at",) if not sort else None,
-                max_results=limit,
-                skip_backend_when_metadata_satisfies_limit=True,
-                include_metadata_for_forced_backend=True,
-            )
-        except Exception as err:
-            rprint(
-                f"[yellow]Search backend error, falling back to metadata search: {err}[/yellow]",
-                file=sys.stderr,
-            )
-            queryset = apply_snapshot_search(queryset, query, search_mode="meta")
-
-    if sort:
-        queryset = queryset.order_by(sort)
-
-    return queryset
+    return Snapshot.objects.order_by("-created_at").search(**kwargs)
 
 
 def list_snapshots(
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
     csv: str | None = None,
+    as_json: bool = False,
+    as_html: bool = False,
     with_headers: bool = False,
-    search: str | None = None,
-    query: str | None = None,
+    **kwargs,
 ) -> int:
     """
     List Snapshots as JSONL with optional filters.
@@ -252,96 +292,71 @@ def list_snapshots(
         0: Success (even if no results)
     """
     from archivebox.misc.jsonl import write_record
-    from archivebox.core.models import Snapshot
 
-    if with_headers and not csv:
-        rprint("[red]--with-headers requires --csv[/red]", file=sys.stderr)
+    output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
+    if output_formats > 1:
+        rprint("[red]Choose only one output format: --csv, --json, or --html[/red]", file=sys.stderr)
+        return 2
+    if with_headers and not output_formats:
+        rprint("[red]--with-headers requires --csv, --json, or --html[/red]", file=sys.stderr)
         return 2
 
-    is_tty = sys.stdout.isatty() and not csv
+    is_tty = sys.stdout.isatty() and not output_formats
 
-    queryset = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=query,
-        limit=limit,
-    )
-
-    if not is_tty:
-        if limit:
-            limited_ids = list(queryset.values_list("id", flat=True)[:limit])
-            preserved_order = Case(
-                *(When(id=snapshot_id, then=position) for position, snapshot_id in enumerate(limited_ids)),
-                output_field=IntegerField(),
-            )
-            queryset = Snapshot.objects.filter(id__in=limited_ids).order_by(preserved_order)
-        queryset = queryset.prefetch_related("tags")
-    elif limit:
-        queryset = queryset[:limit]
+    try:
+        queryset = build_snapshot_queryset(**kwargs)
+    except ValueError as err:
+        rprint(f"[red]{err}[/red]", file=sys.stderr)
+        return 2
 
     count = 0
+    if as_json:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_json(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    if as_html:
+        queryset = queryset.prefetch_related("tags")
+        output = queryset.to_html(with_headers=with_headers)
+        sys.stdout.write(output)
+        if output and not output.endswith("\n"):
+            sys.stdout.write("\n")
+        rprint(f"[dim]Listed {queryset.count()} snapshots[/dim]", file=sys.stderr)
+        return 0
+
     if csv:
         cols = [col.strip() for col in csv.split(",") if col.strip()]
         if not cols:
             rprint("[red]No CSV columns provided[/red]", file=sys.stderr)
             return 2
-        rows: list[str] = []
         if with_headers:
-            rows.append(",".join(cols))
-        simple_cols = {
-            "id",
-            "crawl_id",
-            "url",
-            "title",
-            "timestamp",
-            "depth",
-            "status",
-            "fs_version",
-            "bookmarked_at",
-            "created_at",
-            "modified_at",
-            "retry_at",
-            "downloaded_at",
-        }
-        from archivebox.misc.util import to_json
-
-        for snapshot in queryset.iterator(chunk_size=500):
-            if set(cols).issubset(simple_cols):
-                rows.append(
-                    ",".join(
-                        to_json(
-                            value.isoformat() if hasattr((value := getattr(snapshot, col, "")), "isoformat") else value,
-                            indent=None,
-                        )
-                        for col in cols
-                    ),
-                )
-            else:
-                rows.append(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write(",".join(cols))
+            sys.stdout.write("\n")
+        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+            sys.stdout.write(snapshot.to_csv(cols=cols, separator=","))
+            sys.stdout.write("\n")
             count += 1
-        output = "\n".join(rows)
-        if output:
-            sys.stdout.write(output)
-            if not output.endswith("\n"):
-                sys.stdout.write("\n")
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0
 
-    for snapshot in queryset:
-        if is_tty:
-            status_color = {
-                "queued": "yellow",
-                "started": "blue",
-                "sealed": "green",
-            }.get(snapshot.status, "dim")
-            rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
-        else:
-            write_record(snapshot.to_json())
+    if not is_tty:
+        for snapshot_json in iter_snapshot_json(queryset):
+            write_record(snapshot_json)
+            count += 1
+        rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
+        return 0
+
+    for snapshot in queryset.iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
+        status_color = {
+            "queued": "yellow",
+            "started": "blue",
+            "sealed": "green",
+        }.get(snapshot.status, "dim")
+        rprint(f"[{status_color}]{snapshot.status:8}[/{status_color}] [dim]{snapshot.id}[/dim] {snapshot.url[:60]}")
         count += 1
 
     rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
@@ -500,46 +515,11 @@ def create_cmd(urls: tuple, tag: str, status: str, depth: int):
 
 
 @main.command("list")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of results")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--csv", "-C", type=str, help="Print output as CSV with the provided fields, e.g.: timestamp,url,title")
-@click.option("--with-headers", is_flag=True, help="Include column headers in structured output")
-@click.option("--search", help="Search mode to use for the query")
-@click.argument("query", nargs=-1)
-def list_cmd(
-    status: str | None,
-    url__icontains: str | None,
-    url__istartswith: str | None,
-    tag: str | None,
-    crawl_id: str | None,
-    limit: int | None,
-    sort: str | None,
-    csv: str | None,
-    with_headers: bool,
-    search: str | None,
-    query: tuple[str, ...],
-):
+@snapshot_output_options
+@snapshot_filter_options(default_filter_type="substring")
+def list_cmd(**kwargs):
     """List Snapshots as JSONL."""
-    sys.exit(
-        list_snapshots(
-            status=status,
-            url__icontains=url__icontains,
-            url__istartswith=url__istartswith,
-            tag=tag,
-            crawl_id=crawl_id,
-            limit=limit,
-            sort=sort,
-            csv=csv,
-            with_headers=with_headers,
-            search=search,
-            query=" ".join(query),
-        ),
-    )
+    sys.exit(list_snapshots(**kwargs))
 
 
 @main.command("update")

archivebox/cli/archivebox_snapshot_compat.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-__package__ = "archivebox.cli"
-__command__ = "archivebox snapshot"
-
-import sys
-
-import rich_click as click
-
-from archivebox.cli.archivebox_snapshot import create_snapshots
-
-
-@click.command(context_settings={"ignore_unknown_options": True})
-@click.option("--tag", "-t", default="", help="Comma-separated tags to add")
-@click.option("--status", "-s", default="queued", help="Initial status (default: queued)")
-@click.option("--depth", "-d", type=int, default=0, help="Crawl depth (default: 0)")
-@click.argument("urls", nargs=-1)
-def main(tag: str, status: str, depth: int, urls: tuple[str, ...]):
-    """Backwards-compatible `archivebox snapshot URL...` entrypoint."""
-    sys.exit(create_snapshots(urls, tag=tag, status=status, depth=depth))
-
-
-if __name__ == "__main__":
-    main()

archivebox/cli/archivebox_status.py

@@ -8,7 +8,7 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize
@@ -18,7 +18,7 @@ MAX_STATUS_FS_DIR_SCAN = 5000
 
 
 @enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
+def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
     """Print out some info and statistics about the archive collection"""
 
     from django.contrib.auth import get_user_model
@@ -38,18 +38,24 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     snapshots_qs = Snapshot.objects.all()
     num_sql_links = snapshots_qs.count()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     legacy_snapshot_dirs = []
     if archive_dir.exists():
         legacy_snapshot_dirs = [
             entry for entry in archive_dir.iterdir() if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry)
         ]
-    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
+    from archivebox.misc.db import database_display_location, is_postgres
+
+    index_location = database_display_location() if is_postgres() else CONSTANTS.SQL_INDEX_FILENAME
+    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {index_location})")
     print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = config.USERS_DIR
-    scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
+    users_dir = CONSTANTS.USERS_DIR
+    # The current users layout is nested below archive/, so scanning both
+    # archive/ and archive/users/ recursively counts every current snapshot
+    # twice. A single archive root includes both legacy and current layouts.
+    scan_roots = [archive_dir] if archive_dir.exists() else []
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")
     do_precise_fs_scan = num_sql_links <= MAX_STATUS_FS_DIR_SCAN
@@ -105,11 +111,11 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     if num_indexed:
         print("    [violet]Hint:[/violet] You can list snapshots by status like so:")
-        print("        [green]archivebox list --status=<status>  (e.g. archived, queued, etc.)[/green]")
+        print("        [green]archivebox list --status=<status>  (e.g. sealed, queued, etc.)[/green]")
 
     if orphaned_dirs:
         print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
-        print("        [green]archivebox init[/green]")
+        print("        [green]archivebox update[/green]")
 
     print()
     print("[green]\\[*] Scanning recent archive changes and user logins:[/green]")

archivebox/cli/archivebox_update.py

@@ -15,6 +15,7 @@ from pathlib import Path
 import rich_click as click
 
 from archivebox.misc.util import enforce_types, docstring
+from archivebox.cli.archivebox_snapshot import snapshot_filter_options
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -32,66 +33,36 @@ def _get_snapshot_crawl(snapshot: Snapshot) -> Crawl | None:
 
 
 def _get_search_indexing_plugins() -> list[str]:
-    from abx_dl.models import discover_plugins
+    from archivebox.config.common import get_config
+    from archivebox.plugins.hooks import discover_hooks
     from archivebox.plugins.discovery import get_search_backends
 
     available_backends = set(get_search_backends())
-    plugins = discover_plugins()
     return sorted(
         plugin_name
-        for plugin_name, plugin in plugins.items()
-        if plugin_name.startswith("search_backend_")
-        and plugin_name.removeprefix("search_backend_") in available_backends
-        and any("Snapshot" in hook.name and "index" in hook.name.lower() for hook in plugin.hooks)
+        for plugin_name in {
+            hook.parent.name
+            for hook in discover_hooks("Snapshot", config=get_config())
+            if hook.parent.name.startswith("search_backend_") and "index" in hook.name.lower()
+        }
+        if plugin_name.startswith("search_backend_") and plugin_name.removeprefix("search_backend_") in available_backends
     )
 
 
 def _build_filtered_snapshots_queryset(
-    *,
-    filter_patterns: Iterable[str],
-    filter_type: str,
-    status: str | None = None,
-    url__icontains: str | None = None,
-    url__istartswith: str | None = None,
-    tag: str | None = None,
-    crawl_id: str | None = None,
-    limit: int | None = None,
-    sort: str | None = None,
-    search: str | None = None,
-    before: float | None = None,
-    after: float | None = None,
-    resume: str | None = None,
+    **kwargs,
 ):
-    from datetime import datetime
+    from archivebox.core.models import Snapshot
     from archivebox.cli.archivebox_snapshot import build_snapshot_queryset
 
-    filter_patterns = tuple(filter_patterns)
-    snapshots = build_snapshot_queryset(
-        status=status,
-        url__icontains=url__icontains,
-        url__istartswith=url__istartswith,
-        tag=tag,
-        crawl_id=crawl_id,
-        sort=sort,
-        search=search,
-        query=" ".join(filter_patterns) if search else None,
-    )
-
-    if filter_patterns and not search:
-        snapshots = snapshots.filter_by_patterns(list(filter_patterns), filter_type)
-
-    if before:
-        snapshots = snapshots.filter(bookmarked_at__lt=datetime.fromtimestamp(before))
-    if after:
-        snapshots = snapshots.filter(bookmarked_at__gt=datetime.fromtimestamp(after))
-    if resume:
-        snapshots = snapshots.filter(timestamp__lte=resume)
-    if not sort:
-        snapshots = snapshots.order_by("-timestamp")
+    limit = kwargs.pop("limit", None)
+    snapshots = build_snapshot_queryset(**kwargs)
+    if kwargs.get("resume"):
+        snapshots = snapshots.filter(timestamp__lte=kwargs["resume"])
     snapshots = snapshots.select_related("crawl")
-    if limit:
-        limited_ids = list(snapshots.values_list("id", flat=True)[:limit])
-        snapshots = snapshots.model.objects.filter(id__in=limited_ids).select_related("crawl")
+    if limit is not None and limit > 0:
+        snapshot_ids = list(snapshots.values_list("id", flat=True)[:limit])
+        snapshots = Snapshot.objects.filter(id__in=snapshot_ids).select_related("crawl")
 
     return snapshots
 
@@ -105,9 +76,22 @@ def reindex_snapshots(
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from abx_dl.models import discover_plugins
 
-    stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    # Search backfill is the one maintenance hook allowed to execute without
+    # reopening a Snapshot. Restrict that exception to already-sealed rows;
+    # every open lifecycle state remains owned by the normal state machine.
+    snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
+
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
+    plugins_by_name = discover_plugins(runtime="archivebox")
+    required_hooks_by_plugin = {
+        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
+        for plugin_name in search_plugins
+        if plugin_name in plugins_by_name
+    }
 
     total = snapshots.count()
     print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
@@ -118,6 +102,31 @@ def reindex_snapshots(
         if wait_for_turn:
             wait_for_turn()
         batch_records = list(records)
+        snapshot_ids = {record["snapshot_id"] for record in batch_records}
+        plugin_names = {record["plugin"] for record in batch_records}
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=snapshot_ids,
+                plugin__in=plugin_names,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        records_to_queue = []
+        for record in batch_records:
+            snapshot_id = record["snapshot_id"]
+            plugin_name = record["plugin"]
+            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
+            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
+                stats["skipped_queued"] += 1
+                continue
+            records_to_queue.append(record)
+        if not records_to_queue:
+            print(
+                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
+            )
+            records.clear()
+            return
         # `archivebox update --index-only` intentionally breaks the usual
         # "runner discovers work" rule by inserting synthetic queued
         # ArchiveResult rows for search backends. run_plugins() keeps this as
@@ -127,15 +136,17 @@ def reindex_snapshots(
         # finish.
         exit_code = run_plugins(
             args=(),
-            records=batch_records,
+            records=records_to_queue,
             wait=False,
             emit_results=False,
             show_progress=False,
+            preserve_queued=True,
         )
         if exit_code != 0:
             raise SystemExit(exit_code)
+        stats["queued"] += len(records_to_queue)
         print(
-            f"    [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator",
+            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
         )
         records.clear()
 
@@ -156,7 +167,7 @@ def reindex_snapshots(
                         "plugin": plugin_name,
                     },
                 )
-                stats["queued"] += 1
+                stats["requested"] += 1
             if len(records) >= batch_size:
                 run_batch()
         except KeyboardInterrupt as err:
@@ -219,27 +230,39 @@ def update(
     from archivebox.machine.models import Process
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals, raise_if_shutdown_requested
     from archivebox.core.takeover_util import (
-        command_owns_runtime_stack,
+        command_owns_foreground_runner,
         current_command,
-        ensure_daemon_stack,
-        standby_until_runtime_stack_needed,
+        foreground_runner_owner,
+        standby_until_foreground_runner_needed,
     )
-    from archivebox.workers.supervisord_util import run_runner_worker, stop_existing_supervisord_process, stop_own_supervisord_process
+    from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
 
     command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
 
+    def still_owns_foreground_runner() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
     def wait_for_turn() -> None:
         raise_if_shutdown_requested()
-        standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+        standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
     def run_scoped_runner(*args: str) -> None:
         while True:
             wait_for_turn()
-            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
+            exit_code = run_runner_worker(
+                list(args),
+                name=f"worker_runner_update_{os.getpid()}",
+                keep_running=still_owns_foreground_runner,
+            )
             if exit_code == 0:
                 return
-            if not command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR):
+            if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
                 continue
             raise SystemExit(exit_code)
 
@@ -263,8 +286,6 @@ def update(
 
     try:
         wait_for_turn()
-        if stop_daemon_stack:
-            stop_existing_supervisord_process()
 
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             while True:
@@ -272,6 +293,9 @@ def update(
                 do_index = index_only or not migrate_only
                 do_run_until_idle = do_migrate or do_index
                 ran_post_migrate_runner = False
+                full_update_empty = False
+                maintenance_work_queued = False
+                runner_work_queued = False
 
                 if do_migrate:
                     if (
@@ -308,6 +332,8 @@ def update(
                         )
                         print_stats(stats)
                         touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        maintenance_work_queued = stats.get("queued", 0) > 0
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
                     else:
                         stats_combined = {"phase1": {}, "phase2": {}}
 
@@ -324,74 +350,155 @@ def update(
                             wait_for_turn=wait_for_turn,
                         )
                         print_combined_stats(stats_combined)
+                        full_update_empty = (
+                            stats_combined["phase1"].get("processed", 0) == 0 and stats_combined["phase2"].get("snapshots", 0) == 0
+                        )
+                        maintenance_work_queued = any(
+                            (
+                                stats_combined["phase1"].get("queued", 0),
+                                stats_combined["phase2"].get("queued", 0),
+                                stats_combined["phase2"].get("crawls_sealed", 0),
+                            ),
+                        )
+                        runner_work_queued = runner_work_queued or maintenance_work_queued
 
                     if do_run_until_idle:
                         # Filesystem migration is maintenance on existing
                         # Snapshot rows: Snapshot.save() moves archive/<ts> to
                         # the current output_dir and preserves the lifecycle
                         # status. Drain those retry_at ticks before queuing
-                        # search backfill below. Otherwise the sealed/paused
+                        # search backfill below. Otherwise the sealed search
                         # runner branch correctly sees queued ArchiveResult
                         # rows first, runs the targeted plugins, and may leave
                         # the fs_version maintenance tick hidden behind that
                         # plugin work until another update pass.
-                        print("[*] Phase 3: Running filesystem maintenance until idle...")
-                        if is_filtered_update:
+                        if full_update_empty:
+                            print("[*] No snapshots or legacy archive directories found; skipping filesystem maintenance runner.")
+                        elif not maintenance_work_queued:
+                            print("[*] No filesystem maintenance work queued; skipping filesystem maintenance runner.")
+                        else:
+                            print("[*] Phase 3: Running filesystem maintenance until idle...")
+                        if full_update_empty:
+                            pass
+                        elif not maintenance_work_queued:
+                            pass
+                        elif is_filtered_update:
                             if not touched_snapshot_ids:
                                 print("[*] No matching snapshots queued work for the runner.")
                             for snapshot_id in sorted(touched_snapshot_ids):
                                 run_scoped_runner("--snapshot-id", snapshot_id)
                         else:
-                            run_scoped_runner("--maintenance-only")
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
                         ran_post_migrate_runner = True
 
                 if do_index:
-                    ensure_daemon_stack(reason="search indexing")
-                    search_plugins = _get_search_indexing_plugins()
-                    if not search_plugins:
-                        print("[*] No search indexing plugins are available, nothing to backfill.")
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping search indexing backfill.")
                     else:
-                        snapshots = _build_filtered_snapshots_queryset(
-                            filter_patterns=filter_patterns,
-                            filter_type=filter_type,
-                            status=status,
-                            url__icontains=url__icontains,
-                            url__istartswith=url__istartswith,
-                            tag=tag,
-                            crawl_id=crawl_id,
-                            limit=limit,
-                            sort=sort,
-                            search=search,
-                            before=before,
-                            after=after,
-                            resume=resume,
-                        )
-                        stats = reindex_snapshots(
-                            snapshots,
-                            search_plugins=search_plugins,
-                            batch_size=batch_size,
-                            collect_ids=is_filtered_update,
-                            wait_for_turn=wait_for_turn,
-                        )
-                        print_index_stats(stats)
-                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        search_plugins = _get_search_indexing_plugins()
+                        if not search_plugins:
+                            print("[*] No search indexing plugins are available, nothing to backfill.")
+                        else:
+                            snapshots = _build_filtered_snapshots_queryset(
+                                filter_patterns=filter_patterns,
+                                filter_type=filter_type,
+                                status=status,
+                                url__icontains=url__icontains,
+                                url__istartswith=url__istartswith,
+                                tag=tag,
+                                crawl_id=crawl_id,
+                                limit=limit,
+                                sort=sort,
+                                search=search,
+                                before=before,
+                                after=after,
+                                resume=resume,
+                            )
+                            from django.db.models import Exists, OuterRef, Q
+                            from django.utils import timezone
+                            from archivebox.core.models import ArchiveResult, Snapshot
+
+                            scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                            queued_index_results = ArchiveResult.objects.filter(
+                                status=ArchiveResult.StatusChoices.QUEUED,
+                                plugin__in=search_plugins,
+                            )
+                            if scoped_snapshot_ids is not None:
+                                queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                            if queued_index_results.exists():
+                                runner_work_queued = True
+                                now = timezone.now()
+                                queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                                snapshots_to_wake = (
+                                    Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
+                                    .annotate(
+                                        has_queued_index_result=Exists(queued_result_for_snapshot),
+                                    )
+                                    .filter(
+                                        has_queued_index_result=True,
+                                    )
+                                    .filter(
+                                        Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                    )
+                                )
+                                if scoped_snapshot_ids is not None:
+                                    snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                                woken_count = snapshots_to_wake.update(
+                                    retry_at=now,
+                                    modified_at=now,
+                                )
+                                print(
+                                    "[*] Existing queued search index jobs found; "
+                                    f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                                )
+                            else:
+                                collect_index_ids = (
+                                    is_filtered_update
+                                    or foreground_runner_owner(
+                                        data_dir=CONSTANTS.DATA_DIR,
+                                        exclude_id=command.id,
+                                    )
+                                    is not None
+                                )
+                                stats = reindex_snapshots(
+                                    snapshots,
+                                    search_plugins=search_plugins,
+                                    batch_size=batch_size,
+                                    collect_ids=collect_index_ids,
+                                    wait_for_turn=wait_for_turn,
+                                )
+                                print_index_stats(stats)
+                                touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                                runner_work_queued = runner_work_queued or stats["queued"] > 0
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
-                    # ArchiveResult rows without reopening sealed/paused
-                    # snapshots. This second runner pass drains those plugin
+                    # ArchiveResult rows without reopening sealed snapshots.
+                    # This second runner pass drains those plugin
                     # rows after filesystem maintenance has had its own turn.
                     # For a normal unfiltered `archivebox update`, keep the
                     # historical final pass broad enough to resume genuinely
                     # queued/interrupted crawl work after maintenance is done.
-                    print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
-                    if is_filtered_update:
+                    if full_update_empty:
+                        print("[*] No snapshots found; skipping queued/interrupted crawl runner.")
+                    elif not runner_work_queued:
+                        print("[*] No queued/interrupted crawl work found; skipping queued/interrupted crawl runner.")
+                    else:
+                        print("[*] Phase 3: Running queued/interrupted crawl work until idle...")
+                    if full_update_empty:
+                        pass
+                    elif not runner_work_queued:
+                        pass
+                    elif touched_snapshot_ids and is_filtered_update:
                         if not touched_snapshot_ids:
                             print("[*] No matching snapshots queued work for the runner.")
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     else:
-                        run_scoped_runner(*(["--maintenance-only"] if index_only or migrate_only else []))
+                        run_scoped_runner(
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
+                        )
 
                 if not continuous:
                     break
@@ -401,7 +508,7 @@ def update(
                 resume = None
     except (KeyboardInterrupt, asyncio.CancelledError) as err:
         exit_code = 130
-        exact_resume = getattr(err, "archivebox_resume", None)
+        exact_resume = err.__dict__.get("archivebox_resume")
         resume_cmd = ["archivebox", "update"]
         if migrate_only:
             resume_cmd.append("--migrate-only")
@@ -464,7 +571,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     1:1 mapping between DB and filesystem.
     """
     from archivebox.core.models import Snapshot
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.crawls.models import Crawl
     from django.utils import timezone
 
@@ -473,8 +580,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     crawl_url_sets: dict[str, set[str]] = {}
     dirty_crawl_ids: set[str] = set()
 
-    runtime_config = get_config()
-    archive_dir = runtime_config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     if not archive_dir.exists():
         return stats
 
@@ -639,7 +745,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
         "updated_db": 0,
         "queued": 0,
         "sealed": 0,
-        "crawls_queued": 0,
+        "crawls_sealed": 0,
     }
     current_fs_version = Snapshot._fs_current_version()
 
@@ -647,6 +753,7 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     if resume:
         queryset = queryset.filter(timestamp__lte=resume)
     total = queryset.count()
+    stats["snapshots"] = total
     print(f"[*] Processing {total} snapshots from database (most recent first)...")
 
     def update_in_batches(rows, *, label: str, **updates) -> int:
@@ -745,7 +852,10 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
     queue_stale_fs_batch()
 
     now = timezone.now()
-    stats["crawls_queued"] = (
+    # Crawls with no open child snapshots are already finished. Seal them here
+    # instead of waking the foreground runner; otherwise migration/update can
+    # accidentally re-enter full crawl execution for historical rows.
+    stats["crawls_sealed"] = (
         Crawl.objects.filter(
             status__in=Crawl.RUNNABLE_STATES,
         )
@@ -753,11 +863,12 @@ def process_all_db_snapshots(batch_size: int = 500, resume: str | None = None, w
             snapshot_set__status__in=Snapshot.OPEN_STATES,
         )
         .update(
-            retry_at=now,
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
             modified_at=now,
         )
     )
-    stats["updated_db"] += stats["crawls_queued"]
+    stats["updated_db"] += stats["crawls_sealed"]
     return stats
 
 
@@ -895,7 +1006,7 @@ Phase 2 (Process DB):
   Updated JSON:     {s2.get("updated_json", 0)}
   Updated DB rows:  {s2.get("updated_db", 0)}
   Sealed snapshots: {s2.get("sealed", 0)}
-  Queued crawls:    {s2.get("crawls_queued", 0)}
+  Sealed crawls:    {s2.get("crawls_sealed", 0)}
 """)
 
 
@@ -905,28 +1016,19 @@ def print_index_stats(stats: dict[str, Any]) -> None:
     print(f"""
 [green]Search Reindex Complete[/green]
   Scanned rows:      {stats["processed"]}
+  Requested jobs:    {stats.get("requested", stats["queued"])}
   Queued index jobs: {stats["queued"]}
+  Already queued:    {stats.get("skipped_queued", 0)}
 """)
 
 
 @click.command()
 @click.option("--resume", type=str, help="Resume from timestamp")
-@click.option("--status", "-s", help="Filter by status (queued, started, sealed)")
-@click.option("--url__icontains", help="Filter by URL contains")
-@click.option("--url__istartswith", help="Filter by URL starts with")
-@click.option("--tag", "-t", help="Filter by tag name")
-@click.option("--crawl-id", help="Filter by crawl ID")
-@click.option("--limit", "-n", type=int, help="Limit number of snapshots to update")
-@click.option("--sort", "-o", type=str, help="Field to sort by, e.g. url, created_at, bookmarked_at, downloaded_at")
-@click.option("--search", help="Search mode to use for positional query")
-@click.option("--before", type=float, help="Only snapshots before timestamp")
-@click.option("--after", type=float, help="Only snapshots after timestamp")
-@click.option("--filter-type", type=click.Choice(["exact", "substring", "regex", "domain", "tag", "timestamp"]), default="exact")
 @click.option("--batch-size", type=int, default=500, help="Commit every N records")
 @click.option("--continuous", is_flag=True, help="Run continuously as background worker")
 @click.option("--index-only", is_flag=True, help="Backfill available search indexes from existing archived content")
 @click.option("--migrate-only", is_flag=True, help="Only migrate filesystem and update database/index state")
-@click.argument("filter_patterns", nargs=-1)
+@snapshot_filter_options(default_filter_type="exact")
 @docstring(update.__doc__)
 def main(**kwargs):
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
@@ -934,6 +1036,8 @@ def main(**kwargs):
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
             update(**kwargs)
+    except ValueError as err:
+        raise click.BadParameter(str(err), param_hint="--status") from err
     except KeyboardInterrupt:
         raise SystemExit(130) from None
 

archivebox/cli/archivebox_version.py

@@ -4,7 +4,10 @@ __package__ = "archivebox.cli"
 
 import sys
 import os
+import json
+import asyncio
 import platform
+import re
 from pathlib import Path
 from collections.abc import Iterable
 
@@ -29,10 +32,8 @@ def _format_binary_abspath(
 
     candidate_bases: tuple[tuple[Path, str], ...] = (
         (pwd, "./"),
-        (lib_dir, "LIB_DIR/"),
-        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
+        (lib_dir, "ABXPKG_LIB_DIR/"),
         (personas_dir, "PERSONAS_DIR/"),
-        (Path(os.environ.get("PERSONAS_DIR", "")), "PERSONAS_DIR/") if os.environ.get("PERSONAS_DIR") else (Path(), ""),
         (home, "~/"),
     )
 
@@ -48,8 +49,8 @@ def _format_binary_abspath(
             relative_str = relative.as_posix()
             if prefix == "./":
                 return "." if not relative_str else f"./{relative_str}"
-            if prefix == "LIB_DIR/":
-                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
             if prefix == "PERSONAS_DIR/":
                 return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
             return "~" if not relative_str else f"~/{relative_str}"
@@ -60,10 +61,10 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    if abspath.startswith("LIB_DIR/"):
-        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
-    if abspath == "LIB_DIR":
-        return Text("LIB_DIR", style="bright_blue")
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
     if abspath.startswith("PERSONAS_DIR/"):
         return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
     if abspath == "PERSONAS_DIR":
@@ -79,6 +80,37 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
+def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
+    if not installed or not installed.is_valid or not installed.version:
+        return False
+    try:
+        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
+        if not abspath.exists():
+            return False
+        if installed.binprovider not in {"", "env", "apt", "brew"}:
+            abspath.relative_to(lib_dir)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
+def _binary_row_dedupe_key(
+    *,
+    display_name: str,
+    valid: bool,
+    version: str,
+    provider: str,
+    abspath: str,
+) -> tuple[str, str, str, str]:
+    if not valid:
+        return (display_name, "", "", "")
+    try:
+        resolved_abspath = Path(abspath).expanduser().resolve(strict=False).as_posix()
+    except Exception:
+        resolved_abspath = abspath
+    return (display_name, provider, version, resolved_abspath)
+
+
 @enforce_types
 def version(
     quiet: bool = False,
@@ -95,13 +127,16 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
+    from rich.live import Live
+    from abx_dl.tables import binary_dependency_status, binary_dependency_table
 
     from archivebox.config import CONSTANTS
     from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
     from archivebox.config.paths import get_data_locations, get_code_locations
+    from archivebox.misc.checks import is_archivebox_source_root
     from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.common import get_config
+    from archivebox.config.common import get_config, normalize_runtime_config
 
     console = Console()
     prnt = console.print
@@ -134,7 +169,7 @@ def version(
     try:
         DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
             f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
             f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
             f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -142,7 +177,7 @@ def version(
         )
     except Exception:
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
         )
 
     prnt(
@@ -155,7 +190,38 @@ def version(
     )
     prnt()
 
-    if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    in_data_dir = (
+        not is_archivebox_source_root(CONSTANTS.DATA_DIR)
+        and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+        and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
+    )
+    if isinstance(binaries, str):
+        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+    else:
+        requested_names = {name for name in (binaries or ()) if name}
+
+    def binary_is_requested(logical_name: str, actual_name: str, display_name: str) -> bool:
+        return not requested_names or bool({logical_name, actual_name, display_name} & requested_names)
+
+    def plugin_may_have_requested_binary(plugin) -> bool:
+        if not requested_names:
+            return True
+        aliases = {plugin.name}
+        for required_binary in plugin.config.required_binaries:
+            raw_name = str(required_binary.name)
+            aliases.add(raw_name)
+            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", raw_name)
+            if template_match:
+                config_key = template_match.group(1)
+                aliases.add(config_key)
+                aliases.add(config_key.removesuffix("_BINARY"))
+                default = plugin.config.properties.get(config_key, {}).get("default")
+                if default:
+                    aliases.add(str(default))
+        normalized_aliases = {alias.lower() for alias in aliases}
+        return bool({name.lower() for name in requested_names} & normalized_aliases)
+
+    if not in_data_dir:
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -175,93 +241,260 @@ def version(
                 subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
             ),
         )
+    prnt()
+
+    if not in_data_dir and not requested_names:
+        prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
+        prnt("", "[grey53]Dependency checks require an initialized collection DATA_DIR.[/grey53]")
+        prnt()
+        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
+        try:
+            for name, path in get_code_locations().items():
+                if isinstance(name, str) and isinstance(path, dict):
+                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+        except Exception as e:
+            prnt(f"  [red]Error getting code locations: {e}[/red]")
+        prnt()
+        prnt("[red][i] Data locations:[/red] (not in a data directory)")
         prnt()
         return []
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
+    seen_failures: set[str] = set()
+    seen_rows: set[tuple[str, str, str, str]] = set()
 
-    # Setup Django before importing models
+    from archivebox.plugins.discovery import get_enabled_plugins
+    from abx_dl.config import get_required_binary_requests
+    from abx_dl.dependencies import resolve_binary_requests
+    from abx_dl.models import discover_plugins, filter_plugins
+    from abx_dl.orchestrator import create_bus
+    from abxpkg.binary_service import BinaryEvent, BinaryService
+
+    plugins = discover_plugins(runtime="archivebox")
+    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    enabled_plugin_names = set(enabled_plugins)
+    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    derived_config: dict[str, object] = {}
+    db_binaries = {}
+    db_available = False
+    if in_data_dir:
+        try:
+            from archivebox.config.django import setup_django
+
+            setup_django()
+
+            from archivebox.machine.models import Machine, Binary
+
+            machine = Machine.current()
+            derived_config = normalize_runtime_config(machine.config, json_safe=False)
+            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
+                    db_binaries.setdefault(binary.name, binary)
+            db_available = True
+
+        except Exception as e:
+            prnt()
+            prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
+
+    declared_binary_specs: dict[str, dict[str, object]] = {}
+    for plugin_name, plugin in plugins.items():
+        if not plugin_may_have_requested_binary(plugin):
+            continue
+        plugin_requested = bool(requested_names)
+        plugin_enabled = plugin_name in enabled_plugin_names
+        binary_records = get_required_binary_requests(
+            plugin,
+            plugin.config.required_binaries,
+            overrides=runtime_config,
+            derived_overrides=derived_config,
+            run_output_dir=CONSTANTS.DATA_DIR,
+        )
+        for binary_record in binary_records:
+            actual_name = str(binary_record["name"])
+            logical_name = (
+                Path(actual_name).expanduser().name
+                if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
+                else actual_name
+            )
+            display_name = logical_name
+            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
+                continue
+            if not plugin_enabled and not requested_names:
+                continue
+            if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
+                continue
+            signature = json.dumps(binary_record, sort_keys=True, default=str)
+            declared_binary_specs.setdefault(signature, binary_record)
+
+    loaded_binaries: dict[str, BinaryEvent | None] = {}
+    if declared_binary_specs:
+        binary_bus = create_bus(name="ArchiveBoxVersionBinaryCheck")
+        BinaryService(binary_bus, auto_install=False, lib_dir=config.ABXPKG_LIB_DIR)
+
+        async def resolve_declared_binaries() -> dict[str, BinaryEvent | None]:
+            try:
+                return await resolve_binary_requests(binary_bus, declared_binary_specs)
+            finally:
+                await binary_bus.wait_until_idle()
+                await binary_bus.destroy(clear=False)
+
+        loaded_binaries = asyncio.run(resolve_declared_binaries())
+
+    rows: list[dict[str, object]] = []
+    any_rows = False
+    any_available = False
+    compact_paths = console.is_terminal
+    live_enabled = console.is_terminal
+    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if live_enabled else None
+
+    def emit_row(row: dict[str, object]) -> None:
+        rows.append(row)
+        if live_cm is not None:
+            live_cm.update(binary_dependency_table(rows), refresh=True)
+
+    if live_cm is not None:
+        live_cm.start()
     try:
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-        from archivebox.machine.models import Machine, Binary
-
-        machine = Machine.current()
-
-        if isinstance(binaries, str):
-            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-        else:
-            requested_names = {name for name in (binaries or ()) if name}
-
-        db_binaries: dict[str, Binary] = {}
-        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-            db_binaries.setdefault(binary.name, binary)
-
-        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
-
-        if not all_binary_names:
-            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-        else:
-            any_available = False
-            compact_paths = console.is_terminal
-            for name in all_binary_names:
-                if requested_names and name not in requested_names:
+        for plugin_name, plugin in plugins.items():
+            if not plugin_may_have_requested_binary(plugin):
+                continue
+            plugin_requested = bool(requested_names)
+            plugin_enabled = plugin_name in enabled_plugin_names
+            binary_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+            )
+            for binary_record in binary_records:
+                actual_name = str(binary_record["name"])
+                logical_name = (
+                    Path(actual_name).expanduser().name
+                    if ("/" in actual_name or "\\" in actual_name or actual_name.startswith("~"))
+                    else actual_name
+                )
+                display_name = logical_name
+                if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                     continue
 
-                installed = db_binaries.get(name)
-                if installed and installed.is_valid:
-                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                installed = db_binaries.get(logical_name) if db_available else None
+                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                    abspath = installed.abspath
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    valid = True
+                elif not plugin_enabled and not requested_names:
+                    # `archivebox version` is expected to verify the active runtime, not
+                    # cold-load every optional plugin provider. Migration and status
+                    # checks often run with PLUGINS narrowed to a tiny set; resolving
+                    # disabled plugin binaries there can spend most of the command on
+                    # providers the current collection will never execute.
+                    continue
+                else:
+                    loaded = loaded_binaries[json.dumps(binary_record, sort_keys=True, default=str)]
+                    abspath = loaded.abspath if loaded is not None else ""
+                    version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
+                    provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
+                    valid = loaded is not None
+
+                any_rows = True
+                if valid:
                     display_path = (
                         _format_binary_abspath(
-                            installed.abspath,
+                            abspath,
                             pwd=Path.cwd(),
-                            lib_dir=config.LIB_DIR,
-                            personas_dir=config.PERSONAS_DIR,
+                            lib_dir=config.ABXPKG_LIB_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
                             home=Path.home(),
                         )
                         if compact_paths
-                        else installed.abspath
+                        else abspath
                     )
                     rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    prnt(
-                        "",
-                        "[green]√[/green]",
-                        "",
-                        display_name.ljust(18),
-                        version_str.ljust(16),
-                        provider.ljust(8),
-                        rendered_path,
-                        overflow="ignore",
-                        crop=False,
-                    )
                     any_available = True
-                    continue
+                else:
+                    rendered_path = "[grey53]not installed[/grey53]"
+                    if plugin_enabled and display_name not in seen_failures:
+                        failures.append(display_name)
+                        seen_failures.add(display_name)
 
-                status = (
-                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+                row_key = _binary_row_dedupe_key(
+                    display_name=display_name,
+                    valid=valid,
+                    version=version_str if valid else "-",
+                    provider=provider if valid else "-",
+                    abspath=abspath,
                 )
-                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
-                failures.append(name)
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-            if not any_available:
-                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+                emit_row(
+                    {
+                        "plugin": plugin_name,
+                        "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
+                        "binary": display_name,
+                        "version": version_str if valid else "-",
+                        "provider": provider if valid else "-",
+                        "path": rendered_path,
+                        "style": "" if plugin_enabled else "dim",
+                    },
+                )
 
-        # Show hint if no binaries are installed yet
-        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
-        if not has_any_installed:
-            prnt()
-            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
+        if db_available:
+            for binary_name, installed in db_binaries.items():
+                if requested_names and binary_name not in requested_names:
+                    continue
+                abspath = installed.abspath
+                version_str = (installed.version or "unknown")[:15]
+                provider = (installed.binprovider or "env")[:8]
+                row_key = _binary_row_dedupe_key(
+                    display_name=binary_name,
+                    valid=True,
+                    version=version_str,
+                    provider=provider,
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-    except Exception as e:
-        # Handle database errors gracefully (locked, missing, etc.)
-        prnt()
-        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
-        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                emit_row(
+                    {
+                        "plugin": "(database)",
+                        "status": binary_dependency_status(enabled=True, valid=True),
+                        "binary": binary_name,
+                        "version": version_str,
+                        "provider": provider,
+                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
+                        "style": "dim",
+                    },
+                )
+                any_rows = True
+                any_available = True
+    finally:
+        if live_cm is not None:
+            live_cm.stop()
+        else:
+            prnt(binary_dependency_table(rows))
+
+    if not any_rows:
+        prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
+    elif not any_available:
+        prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
 
     if not binaries:
         # Show code and data locations
@@ -275,7 +508,7 @@ def version(
             prnt(f"  [red]Error getting code locations: {e}[/red]")
 
         prnt()
-        if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
+        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
             prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
             try:
                 for name, path in get_data_locations().items():
@@ -320,9 +553,7 @@ def version(
 )
 @docstring(version.__doc__)
 def main(**kwargs):
-    failures = version(**kwargs)
-    if failures:
-        raise SystemExit(1)
+    version(**kwargs)
 
 
 if __name__ == "__main__":

archivebox/config/collection.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.config"
 import io
 import json
 import os
+from collections.abc import Mapping
 from typing import Any
 
 from archivebox.config.constants import CONSTANTS
@@ -54,7 +55,7 @@ def _coerce_to_str_dict(config: Any) -> dict[str, str]:
     """
     if not config:
         return {}
-    if not hasattr(config, "items"):
+    if not isinstance(config, Mapping):
         return {}
     flat: dict[str, str] = {}
     for key, value in config.items():
@@ -193,6 +194,12 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
             decoded[key] = json.loads(raw)
         except (TypeError, ValueError):
             continue
+    if not str(decoded.get("BASE_URL") or "").strip():
+        from archivebox.config.common import base_url_from_legacy_server_config
+
+        legacy_base_url = base_url_from_legacy_server_config(decoded)
+        if legacy_base_url:
+            decoded["BASE_URL"] = legacy_base_url
     return decoded
 
 
@@ -319,7 +326,7 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
         # validate the updated_config by attempting to re-parse it
         from archivebox.config.common import get_config
 
-        updated_config = get_config().as_dict()
+        updated_config = get_config(include_machine=False).as_dict()
     except BaseException:  # lgtm [py/catch-base-exception]
         # something went horribly wrong, revert to the previous version
         with open(f"{config_path}.bak", encoding="utf-8") as old:

archivebox/config/common.py

@@ -2,34 +2,37 @@ from __future__ import annotations
 
 __package__ = "archivebox.config"
 
+import inspect
 import json
 import os
 import re
 import secrets
-import sys
 import shutil
-from functools import lru_cache
+import sys
 from collections.abc import Mapping
 from datetime import timedelta
-from typing import Any, ClassVar, cast
+from functools import cache, lru_cache
 from pathlib import Path
+from typing import Any, ClassVar, cast
+from urllib.parse import quote, urlparse
 
-from rich.console import Console
+from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from django.db import DatabaseError
 from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
 from pydantic_settings import SettingsConfigDict
-from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from rich.console import Console
 
-from archivebox.config.configset import BaseConfigSet, IniConfigSettingsSource
-from archivebox.config.configset import COMPUTED_CONFIG_KEYS
+from archivebox.config.configset import COMPUTED_CONFIG_KEYS, BaseConfigSet, IniConfigSettingsSource
 
 from .constants import CONSTANTS
 from .ldap import LDAPConfig
-from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
 from .permissions import IN_DOCKER
+from .version import VERSION, get_BUILD_TIME, get_COMMIT_HASH
 
 ConfigOverrides = Mapping[str, object]
 ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
+LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
 ###################### Config ##########################
 
@@ -64,6 +67,50 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
     return None
 
 
+def base_url_from_legacy_server_config(raw_config: Mapping[str, object]) -> str:
+    """Derive the current canonical URL from the pre-0.9 server settings."""
+
+    def normalize(value: object) -> str:
+        raw = str(value or "").strip()
+        if not raw:
+            return ""
+        parsed = urlparse(raw if "://" in raw else f"http://{raw}")
+        return f"{parsed.scheme}://{parsed.netloc}" if parsed.netloc else ""
+
+    for key in ("ARCHIVE_BASE_URL", "ADMIN_BASE_URL"):
+        candidate = normalize(raw_config.get(key))
+        if candidate:
+            return candidate
+
+    listen_host = str(raw_config.get("LISTEN_HOST") or "").strip()
+    if listen_host and listen_host.lower() != "archivebox.localhost:8000":
+        return normalize(listen_host)
+
+    csrf_origins = [normalize(origin) for origin in str(raw_config.get("CSRF_TRUSTED_ORIGINS") or "").split(",")]
+    csrf_origins = [origin for origin in csrf_origins if origin]
+    nondefault_csrf_origins = [origin for origin in csrf_origins if origin != "http://admin.archivebox.localhost:8000"]
+    if len(nondefault_csrf_origins) == 1:
+        return nondefault_csrf_origins[0]
+
+    allowed_hosts = [host.strip() for host in str(raw_config.get("ALLOWED_HOSTS") or "").split(",") if host.strip() != "*"]
+    if len(allowed_hosts) == 1:
+        return normalize(allowed_hosts[0])
+
+    if len(csrf_origins) == 1:
+        return csrf_origins[0]
+    return normalize(listen_host)
+
+
+def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str:
+    for config in configs:
+        if config is None:
+            continue
+        value = config.get("DELETE_AFTER")
+        if value:
+            return str(value)
+    return "0"
+
+
 _SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
 SENSITIVE_CONFIG_VALUE_REDACTED = "********"
 _SCOPE_CRAWL_FROZEN = "crawl_frozen"
@@ -118,8 +165,15 @@ def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
     return redacted
 
 
-def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
-    """Return a JSON-safe config dict suitable for storage or event payloads."""
+def normalize_runtime_config(
+    config: BaseConfigSet | Mapping[str, Any] | str | None,
+    *,
+    only_crawl_execution: bool = False,
+    exclude_runtime_derived: bool = False,
+    exclude_crawl_execution: bool = False,
+    json_safe: bool = True,
+) -> dict[str, Any]:
+    """Return config filtered for runtime/frozen usage, optionally JSON-safe."""
     if config is None:
         return {}
     if isinstance(config, BaseConfigSet):
@@ -128,21 +182,59 @@ def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | N
         config = json.loads(config)
     else:
         config = dict(config)
-    return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
+
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys() if exclude_runtime_derived else frozenset()
+    filtered = {
+        key: value
+        for key, value in config.items()
+        if (
+            value is not None
+            and (not only_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) == _SCOPE_CRAWL_EXECUTION)
+            and (not exclude_runtime_derived or str(key) not in runtime_derived_keys)
+            and (not exclude_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) != _SCOPE_CRAWL_EXECUTION)
+        )
+    }
+    if not json_safe:
+        return filtered
+    return {key: value for key, value in json.loads(json.dumps(filtered, default=str)).items() if value is not None}
 
 
 def build_crawl_config_snapshot(
     *,
-    user: Any = None,
     persona: Any = None,
     overrides: Mapping[str, Any] | None = None,
     base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
 ) -> dict[str, Any]:
-    """Build the frozen runtime config stored on Crawl.config at creation time."""
-    effective = get_config(user=user, persona=persona, base_config=base_config)
-    frozen = effective.for_crawl_frozen()
+    """Build the frozen crawl config stored on Crawl.config at creation time."""
+    explicit_overrides = set(overrides or {})
+    plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
+    effective = get_config(persona=persona, base_config=base_config)
+    frozen = effective.for_crawl_frozen(persona=persona)
+    for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+        value = getattr(effective, key, None)
+        if value is not None:
+            frozen[key] = value
+    if persona is not None:
+        persona_config = persona.get_derived_config()
+        for key in plugin_owned_keys - explicit_overrides:
+            if key in persona_config:
+                frozen.pop(key, None)
     if overrides:
-        frozen = get_config(base_config=frozen, overrides=overrides, include_machine=False).for_crawl_frozen()
+        resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
+        resolved_payload = normalize_runtime_config(resolved)
+        frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+            value = getattr(resolved, key, None)
+            if value is not None:
+                frozen[key] = value
+        for key in plugin_owned_keys & explicit_overrides:
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
+                frozen[key] = resolved_payload[key]
+        if persona is not None:
+            persona_config = persona.get_derived_config()
+            for key in plugin_owned_keys - explicit_overrides:
+                if key in persona_config:
+                    frozen.pop(key, None)
     return frozen
 
 
@@ -187,28 +279,15 @@ class StorageConfig(BaseConfigSet):
     toml_section_header: str = "STORAGE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    # ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
-    ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
-    USERS_DIR: Path = Field(default=CONSTANTS.USERS_DIR)
-    PERSONAS_DIR: Path = Field(default=CONSTANTS.PERSONAS_DIR)
-
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
+    # must be a short path due to unix path length restrictions for socket files (<90 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
     TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
+    # ABXPKG_LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be able to contain executable binaries (up to 5GB size)
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-
-    # LIB_BIN_DIR is an optional human-facing symlink convenience directory.
-    # Runtime lookup must use provider-specific paths under LIB_DIR instead.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-
-    # CUSTOM_TEMPLATES_DIR allows users to override default templates
-    # defaults to DATA_DIR / 'user_templates' but can be configured
-    CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
+    ABXPKG_LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     OUTPUT_PERMISSIONS: str = Field(default="644")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
@@ -227,6 +306,7 @@ class ServerConfig(BaseConfigSet):
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = (
+        "auto",
         "safe-subdomains-fullreplay",
         "safe-onedomain-nojsreplay",
         "unsafe-onedomain-noadmin",
@@ -238,14 +318,12 @@ class ServerConfig(BaseConfigSet):
     BASE_URL: str = Field(default="")
     ALLOWED_HOSTS: str = Field(default="*")
     CSRF_TRUSTED_ORIGINS: str = Field(default="")
-    SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay")
+    SERVER_SECURITY_MODE: str = Field(default="auto")
 
     SNAPSHOTS_PER_PAGE: int = Field(default=50, ge=1)
     FOOTER_INFO: str = Field(
         default="Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.",
     )
-    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
-
     PUBLIC_INDEX: bool = Field(default=True)
     PUBLIC_ADD_VIEW: bool = Field(default=False)
 
@@ -265,11 +343,21 @@ class ServerConfig(BaseConfigSet):
 
     @property
     def USES_SUBDOMAIN_ROUTING(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
+        if self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay":
+            return True
+        if self.SERVER_SECURITY_MODE != "auto":
+            return False
+
+        base_host = (urlparse(self.BASE_URL if "://" in self.BASE_URL else f"//{self.BASE_URL}").hostname or "").lower()
+        if base_host:
+            return base_host.endswith(".localhost")
+
+        bind_host = (urlparse(f"//{self.BIND_ADDR}").hostname or "").lower()
+        return bind_host in {"", "0.0.0.0", "127.0.0.1", "::", "::1", "localhost"}
 
     @property
     def ENABLES_FULL_JS_REPLAY(self) -> bool:
-        return self.SERVER_SECURITY_MODE in (
+        return self.USES_SUBDOMAIN_ROUTING or self.SERVER_SECURITY_MODE in (
             "safe-subdomains-fullreplay",
             "unsafe-onedomain-noadmin",
             "danger-onedomain-fullreplay",
@@ -285,7 +373,7 @@ class ServerConfig(BaseConfigSet):
 
     @property
     def SHOULD_NEUTER_RISKY_REPLAY(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "safe-onedomain-nojsreplay"
+        return self.SERVER_SECURITY_MODE in ("auto", "safe-onedomain-nojsreplay")
 
     @property
     def IS_UNSAFE_MODE(self) -> bool:
@@ -307,7 +395,16 @@ class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
+    DATABASE_ENGINE: str = Field(
+        default="sqlite",
+        alias="ARCHIVEBOX_DATABASE_ENGINE",
+        pattern=r"(?i)^(sqlite|postgres|postgresql)$",
+    )
     DATABASE_NAME: str = Field(default=str(CONSTANTS.DATABASE_FILE), alias="ARCHIVEBOX_DATABASE_NAME")
+    DATABASE_HOST: str = Field(default="127.0.0.1", alias="ARCHIVEBOX_DATABASE_HOST")
+    DATABASE_PORT: int = Field(default=5432, alias="ARCHIVEBOX_DATABASE_PORT", ge=1, le=65535)
+    DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
+    DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
         default="WAL",
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
@@ -355,7 +452,7 @@ class ArchivingConfig(BaseConfigSet):
     URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
 
-    DEFAULT_PERSONA: str = Field(default="Default")
+    DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     PERMISSIONS: str = Field(
         default="public",
         description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.",
@@ -449,7 +546,7 @@ class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
+    SEARCH_BACKEND_ENGINE: str = Field(default="sonic", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
 
 def _plugin_user_config_value(value: Any) -> str:
@@ -464,14 +561,38 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
     return {key: _plugin_user_config_value(value) for key, value in config.items()}
 
 
+def _plugin_input_config(config: Mapping[str, object]) -> dict[str, object]:
+    """Return raw config inputs understood by core or plugin schemas."""
+    input_names = _archivebox_config_input_names()
+    return {str(key): value for key, value in config.items() if str(key) in input_names}
+
+
+def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
+    """Return canonical ``*_ENABLED`` keys explicitly provided by the user.
+
+    ``PLUGINS`` is a selector, not an override for concrete plugin enable flags.
+    If a user passes ``PLUGINS=wget`` and ``SAVE_WGET=False`` together, the
+    alias-resolved ``WGET_ENABLED=False`` must survive the model validator that
+    expands ``PLUGINS`` into concrete booleans.
+    """
+    provided = {str(key) for key in config}
+    explicit: set[str] = set()
+    for key, prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if not key.endswith("_ENABLED") or not isinstance(prop, Mapping):
+            continue
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        input_names = {key, *(str(alias) for alias in aliases if str(alias).strip())} if isinstance(aliases, list) else {key}
+        if provided & input_names:
+            explicit.add(key)
+    return explicit
+
+
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
     from archivebox.plugins.discovery import discover_plugin_configs
 
     schemas: PluginSchemaDocuments = {}
     if BASE_CONFIG_PATH.exists():
-        schemas["base"] = {
-            "properties": json.loads(BASE_CONFIG_PATH.read_text()).get("properties", {}),
-        }
+        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
     schemas.update(discover_plugin_configs())
     return schemas
 
@@ -495,6 +616,10 @@ def _archivebox_config_input_names() -> set[str]:
     for field in ArchiveBoxConfig.model_fields.values():
         if isinstance(field.alias, str):
             names.add(field.alias)
+    for prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).values():
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        if isinstance(aliases, list):
+            names.update(str(alias) for alias in aliases if str(alias).strip())
     return names
 
 
@@ -524,10 +649,6 @@ class ArchiveBoxBaseConfig(
         populate_by_name=True,
     )
 
-    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    ABX_RUNTIME: str = Field(default="archivebox", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    CRAWL_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    SNAP_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
 
     @classmethod
@@ -566,68 +687,183 @@ class ArchiveBoxBaseConfig(
 
     @classmethod
     def _plugin_field_scope(cls, key: str) -> str | None:
-        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        scope = None
+        for schema in PLUGIN_CONFIG_SCHEMAS.values():
             properties = schema.get("properties") if isinstance(schema, dict) else None
             if not isinstance(properties, dict) or key not in properties:
                 continue
             prop_schema = properties.get(key) or {}
             if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
-                return str(prop_schema["x-scope"])
-            if str(plugin_name).startswith("search_backend_"):
-                return _SCOPE_SERVER
-            return _SCOPE_CRAWL_FROZEN
-        return None
+                scope = str(prop_schema["x-scope"])
+            elif scope is None:
+                scope = _SCOPE_CRAWL_FROZEN
+        return scope
 
     @classmethod
+    @cache
     def scope_for_key(cls, key: str) -> str:
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+            properties = schema.get("properties") if isinstance(schema, dict) else None
+            if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties:
+                return _SCOPE_CRAWL_EXECUTION
+        if key.endswith("_BINARY"):
+            return _SCOPE_CRAWL_EXECUTION
         return cls._core_field_scope(key) or cls._plugin_field_scope(key) or _SCOPE_SERVER
 
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _scope_by_key(cls) -> dict[str, str]:
+        return {key: cls.scope_for_key(key) for key in cls.model_fields}
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_frozen_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_FROZEN)
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_runtime_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope in {_SCOPE_CRAWL_FROZEN, _SCOPE_CRAWL_EXECUTION})
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def runtime_derived_config_keys(cls) -> frozenset[str]:
+        runtime_derived_keys = {
+            "ABX_INSTALL_CACHE",
+            "ACTIVE_PERSONA",
+            "CRAWL_DIR",
+            "DEFAULT_PERSONA",
+            "EXTRA_CONTEXT",
+            "PERSONAS_DIR",
+            "SNAP_DIR",
+        }
+        return frozenset(
+            key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_EXECUTION and key in runtime_derived_keys
+        )
+
     def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
-        allowed_scopes = {_SCOPE_CRAWL_FROZEN}
-        if include_execution:
-            allowed_scopes.add(_SCOPE_CRAWL_EXECUTION)
-        return {key: value for key, value in normalize_runtime_config(self).items() if type(self).scope_for_key(key) in allowed_scopes}
+        keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
+        payload = self.model_dump(mode="json")
+        return {key: payload[key] for key in keys if payload.get(key, None) is not None}
 
-    def for_crawl_execution(self) -> dict[str, Any]:
-        """Config safe to pass to crawl/snapshot hook execution."""
-        return self._scoped_config(include_execution=True)
+    def for_crawl(self) -> dict[str, Any]:
+        """Config scoped to crawl execution, without runtime object overlays."""
+        config = self._scoped_config(include_execution=True)
+        for key in type(self).runtime_derived_config_keys():
+            config.pop(key, None)
+        return config
 
-    def for_crawl_frozen(self) -> dict[str, Any]:
+    def for_crawl_frozen(self, *, persona: Any = None) -> dict[str, Any]:
         """Config safe to persist permanently on Crawl.config."""
-        return self._scoped_config(include_execution=False)
+        frozen = self._scoped_config(include_execution=False)
+        if persona is not None:
+            persona_config = dict(persona.config or {})
+            scope_by_key = type(self)._scope_by_key()
+            for key in persona.get_derived_config():
+                if key not in persona_config and scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    frozen.pop(key, None)
+        return frozen
+
+    def for_crawl_runtime(
+        self,
+        *,
+        crawl: Any = None,
+        snapshot: Any = None,
+        persona: Any = None,
+        runtime_overrides: Mapping[str, Any] | None = None,
+        extra_context: Mapping[str, Any] | None = None,
+        crawl_output_dir: Any = None,
+        snapshot_output_dir: Any = None,
+    ) -> dict[str, Any]:
+        """Config payload safe to pass to crawl/snapshot hook execution."""
+        config = self.for_crawl()
+        config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
+        scope_by_key = type(self)._scope_by_key()
+        model_fields = type(self).model_fields
+        # ArchiveBox-only selectors are consumed during model validation to
+        # derive hook-owned *_ENABLED flags. They are stripped below after
+        # crawl/snapshot overlays have had a chance to apply.
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+        if persona is not None:
+            for key, value in persona.get_derived_config().items():
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+
+        if crawl is not None:
+            for key, value in dict(crawl.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["CRAWL_DIR"] = str(crawl_output_dir if crawl_output_dir is not None else crawl.output_dir)
+
+        if snapshot is not None:
+            for key, value in dict(snapshot.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["SNAP_DIR"] = str(snapshot_output_dir if snapshot_output_dir is not None else snapshot.output_dir)
+
+        if runtime_overrides:
+            config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
+
+        # Hooks should only see concrete plugin-local flags, never the
+        # ArchiveBox selectors used to derive them.
+        config.pop("PLUGINS", None)
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+
+        if extra_context:
+            context: dict[str, Any] = {}
+            if config.get("EXTRA_CONTEXT"):
+                parsed_extra_context = json.loads(str(config["EXTRA_CONTEXT"]))
+                if not isinstance(parsed_extra_context, dict):
+                    raise TypeError("EXTRA_CONTEXT must decode to an object")
+                context = parsed_extra_context
+            context.update(dict(extra_context))
+            config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True)
+
+        return config
 
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
-        self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
-
-        archive_dir = self.ARCHIVE_DIR.expanduser()
-        if archive_dir == (CONSTANTS.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME) and self.DATA_DIR != CONSTANTS.DATA_DIR:
-            archive_dir = self.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME
-        if not archive_dir.is_absolute():
-            archive_dir = self.DATA_DIR / archive_dir
-        self.ARCHIVE_DIR = archive_dir.resolve()
-
-        users_dir = self.USERS_DIR.expanduser()
-        if users_dir == (CONSTANTS.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME):
-            users_dir = self.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME
-        if not users_dir.is_absolute():
-            users_dir = self.ARCHIVE_DIR / users_dir
-        self.USERS_DIR = users_dir.resolve()
-
-        lib_dir = self.LIB_DIR.expanduser()
+        lib_dir = self.ABXPKG_LIB_DIR.expanduser()
         if not lib_dir.is_absolute():
-            lib_dir = self.DATA_DIR / lib_dir
-        self.LIB_DIR = lib_dir.resolve()
-
-        lib_bin_dir = self.LIB_BIN_DIR.expanduser()
-        if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
-            lib_bin_dir = self.LIB_DIR / "bin"
-        elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = self.DATA_DIR / lib_bin_dir
-        self.LIB_BIN_DIR = lib_bin_dir.resolve()
+            lib_dir = CONSTANTS.DATA_DIR / lib_dir
+        self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
         return self
 
+    @model_validator(mode="after")
+    def derive_plugin_enabled_config(self):
+        self._derive_plugin_enabled_config(respect_current_enabled=False)
+        return self
+
+    def _derive_plugin_enabled_config(self, *, respect_current_enabled: bool) -> None:
+        plugin_names = _normalize_plugins_config_value(self.PLUGINS)
+        enabled_config_keys = _plugin_enabled_config_keys()
+        disabled_plugins = []
+        if respect_current_enabled:
+            disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
+        selected_plugin_roots = plugin_names
+        if selected_plugin_roots:
+            from abx_dl.models import discover_plugins, filter_plugins
+
+            selected_plugins = set(
+                filter_plugins(
+                    discover_plugins(runtime="archivebox"),
+                    sorted(selected_plugin_roots),
+                    include_providers=True,
+                    disabled_names=disabled_plugins,
+                ),
+            )
+        else:
+            selected_plugins = set()
+        search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
+        if search_backend:
+            selected_plugins.add(f"search_backend_{search_backend}")
+        if respect_current_enabled:
+            selected_plugins.difference_update(disabled_plugins)
+        for plugin_name, enabled_key in enabled_config_keys.items():
+            if plugin_names or plugin_name in selected_plugins:
+                setattr(self, enabled_key, plugin_name in selected_plugins)
+
 
 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
     core_fields = set(ArchiveBoxBaseConfig.model_fields)
@@ -649,15 +885,179 @@ PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
 ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)
 
 
+def _normalize_plugins_config_value(value: Any) -> set[str]:
+    if value is None:
+        return set()
+    if isinstance(value, str):
+        raw = value.strip()
+        if not raw:
+            return set()
+        if raw.startswith("["):
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list):
+                return {str(plugin).strip().lower() for plugin in parsed if str(plugin).strip()}
+        return {plugin.strip().lower() for plugin in raw.split(",") if plugin.strip()}
+    if isinstance(value, (list, tuple, set)):
+        return {str(plugin).strip().lower() for plugin in value if str(plugin).strip()}
+    normalized = str(value).strip().lower()
+    return {normalized} if normalized else set()
+
+
+@lru_cache(maxsize=1)
+def _plugin_enabled_config_keys() -> dict[str, str]:
+    enabled_keys: dict[str, str] = {}
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        enabled_key = f"{str(plugin_name).upper()}_ENABLED"
+        if enabled_key in properties and ArchiveBoxConfig.scope_for_key(enabled_key) == _SCOPE_CRAWL_EXECUTION:
+            enabled_keys[str(plugin_name).lower()] = enabled_key
+    return enabled_keys
+
+
+def get_live_config_url(key: str) -> str:
+    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
+
+
+@lru_cache(maxsize=1)
+def config_field_metadata() -> dict[str, dict[str, Any]]:
+    """Return one centralized metadata map for core and plugin config fields."""
+    metadata: dict[str, dict[str, Any]] = {}
+    for key, field in ArchiveBoxConfig.model_fields.items():
+        if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION or key in ArchiveBoxConfig.computed_config_keys:
+            continue
+        default = field.default
+        try:
+            json.dumps(default)
+        except TypeError:
+            default = str(default)
+        metadata[key] = {
+            "plugin": "archivebox",
+            "section": find_config_section(key),
+            "type": config_field_type(key),
+            "default": default,
+            "description": field.description or "",
+            "scope": ArchiveBoxConfig.scope_for_key(key),
+            "sensitive": is_sensitive_config_key(key),
+        }
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        for key, prop in properties.items():
+            if not isinstance(prop, Mapping):
+                continue
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION:
+                continue
+            metadata[key] = {
+                **metadata.get(key, {}),
+                "plugin": plugin_name,
+                "section": "PLUGINS",
+                "type": prop.get("type", metadata.get(key, {}).get("type", "string")),
+                "default": prop.get("default", metadata.get(key, {}).get("default", "")),
+                "description": prop.get("description", metadata.get(key, {}).get("description", "")),
+                "scope": ArchiveBoxConfig.scope_for_key(key),
+                "sensitive": bool(prop.get("x-sensitive")) or is_sensitive_config_key(key),
+                "schema": dict(prop),
+            }
+    return metadata
+
+
+def find_config_section(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return "CONSTANT"
+    for section_id, section in get_all_configs().items():
+        if key in type(section).model_fields:
+            return section_id
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "PLUGINS"
+    return "DYNAMIC"
+
+
+def find_config_default(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return str(CONSTANTS_CONFIG[key])
+
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return ""
+    default_val = field.default
+    if callable(default_val):
+        default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
+        if default_val.count(")") > default_val.count("("):
+            default_val = default_val[:-1]
+    else:
+        default_val = str(default_val)
+    return default_val
+
+
+def config_field_type(key: str) -> str:
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return "str"
+    annotation = field.annotation
+    try:
+        return annotation.__name__
+    except AttributeError:
+        return str(annotation)
+
+
+def find_config_type(key: str) -> str:
+    return config_field_type(key)
+
+
+def find_config_source(key: str, merged_config: Mapping[str, Any]) -> str:
+    """Determine where a config value comes from."""
+    from archivebox.machine.models import Machine
+
+    machine = Machine.current()
+    if machine.config and key in machine.config:
+        return "Machine"
+
+    if key in os.environ:
+        return "Environment"
+
+    file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+    if key in file_config:
+        return "File"
+
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "Plugin Default"
+
+    return "Default"
+
+
+def get_request_config(request: Any, *, resolve_plugins: bool = False) -> ArchiveBoxBaseConfig:
+    """Return the per-request ArchiveBox config, upgrading to plugin resolution if needed."""
+    request_state = request.__dict__
+    request_config = request_state.get("archivebox_config")
+    request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
+    if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
+        request_config = get_config(resolve_plugins=resolve_plugins)
+        request._archivebox_config_resolves_plugins = resolve_plugins
+    if request_config.SERVER_SECURITY_MODE == "auto":
+        request_host = (urlparse(f"//{request.get_host()}").hostname or "").lower().rstrip(".")
+        effective_mode = "safe-subdomains-fullreplay" if request_host.endswith(".localhost") else "safe-onedomain-nojsreplay"
+        request_config = request_config.model_copy(update={"SERVER_SECURITY_MODE": effective_mode})
+    request.archivebox_config = request_config
+    return request_config
+
+
 def get_config(
     defaults: ConfigOverrides | None = None,
     overrides: ConfigOverrides | None = None,
     base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
     persona: Any = None,
-    user: Any = None,
     crawl: Any = None,
     snapshot: Any = None,
-    archiveresult: Any = None,
     machine: Any = None,
     include_machine: bool = True,
     resolve_plugins: bool = True,
@@ -666,28 +1066,20 @@ def get_config(
     """
     Get merged config from all sources.
 
-    Priority (highest to lowest):
-    1. Explicit overrides
-    2. Per-ArchiveResult config
-    3. Per-snapshot config and output path
-    4. Frozen per-crawl config and output path
-    5. Per-user config (only when resolving outside a crawl)
-    6. Per-persona derived config (only when resolving outside a crawl)
-    7. Current machine derived config (only when resolving outside a crawl)
-    8. Environment variables (only when resolving outside a crawl)
-    9. Config file (ArchiveBox.conf, only when resolving outside a crawl)
-    10. Plugin schema defaults
-    11. Core config defaults
-    """
-    if snapshot is None and archiveresult is not None:
-        snapshot = archiveresult.snapshot
+    Defaults are hydrated by pydantic from core/plugin defaults,
+    ArchiveBox.conf, and environment variables. Persisted Machine/Persona
+    values then apply for live crawl-execution scope, while Crawl/Snapshot
+    rows apply their frozen crawl-scope values. Explicit overrides win last.
 
+    Crawl-execution config is not persisted on Crawl.config. It is rederived
+    from current Machine/Persona state and hydrated process defaults each time.
+    """
     if crawl is None and snapshot is not None:
         crawl = snapshot.crawl
 
     crawl_config_base = crawl is not None and base_config is None
 
-    if include_machine and machine is None and not crawl_config_base:
+    if include_machine and machine is None:
         try:
             from django.apps import apps
 
@@ -695,61 +1087,79 @@ def get_config(
                 from archivebox.machine.models import Machine
 
                 machine = Machine.current()
-        except Exception:
+        except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
             machine = None
 
-    if persona is None and crawl is not None and not crawl_config_base:
+    if persona is None and crawl is not None:
         persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
+    config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
     base_config_payload: ConfigPayload = {}
+    base_config_model = ArchiveBoxConfig()
+
     if crawl_config_base:
-        config_data.update(dict(crawl.config or {}))
+        config_data.update(
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
+        config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
     elif base_config is not None:
         if isinstance(base_config, ArchiveBoxBaseConfig):
             base_config_payload.update(base_config.model_dump(mode="json"))
         else:
             base_config_payload.update(dict(base_config))
-        config_data.update(base_config_payload)
+        config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
-        config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
-        legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
+        config_data.update(
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
+        legacy_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ}
+        legacy_permissions = permissions_from_legacy_public_flags(legacy_config)
         if legacy_permissions:
             config_data["PERMISSIONS"] = legacy_permissions
+        if not str(config_data.get("BASE_URL") or "").strip():
+            legacy_base_url = base_url_from_legacy_server_config(legacy_config)
+            if legacy_base_url:
+                config_data["BASE_URL"] = legacy_base_url
 
     scope_overrides: ConfigPayload = {}
 
-    if not crawl_config_base:
-        if include_machine and machine is not None and machine.config:
-            from archivebox.machine.models import _sanitize_machine_config
+    if include_machine and machine is not None and machine.config:
+        from archivebox.machine.models import _sanitize_machine_config
 
-            scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
+        scope_overrides.update(
+            normalize_runtime_config(
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                only_crawl_execution=crawl_config_base,
+                exclude_runtime_derived=True,
+                json_safe=False,
+            ),
+        )
 
-        if persona is not None:
-            scope_overrides.update(persona.get_derived_config())
-
-        user_config = getattr(user, "config", None)
-        if user_config:
-            scope_overrides.update(user_config)
+    if persona is not None:
+        persona_config = normalize_runtime_config(
+            persona.get_derived_config(),
+            exclude_runtime_derived=not crawl_config_base,
+            json_safe=False,
+        )
+        if crawl_config_base:
+            scope_by_key = ArchiveBoxConfig._scope_by_key()
+            crawl_keys = set(dict(crawl.config or {}))
+            persona_config = {
+                key: value
+                for key, value in persona_config.items()
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION or key not in crawl_keys
+            }
+        scope_overrides.update(persona_config)
 
     if crawl is not None and crawl.config and not crawl_config_base:
-        scope_overrides.update(crawl.config)
-
-    if crawl is not None:
-        if not overrides or "CRAWL_DIR" not in overrides:
-            scope_overrides["CRAWL_DIR"] = crawl.output_dir
+        scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False))
 
     if snapshot is not None and snapshot.config:
-        scope_overrides.update(snapshot.config)
-
-    if snapshot is not None:
-        scope_overrides["SNAP_DIR"] = snapshot.output_dir
-
-    if archiveresult is not None and archiveresult.config:
-        scope_overrides.update(archiveresult.config)
+        scope_overrides.update(normalize_runtime_config(snapshot.config, exclude_crawl_execution=True, json_safe=False))
 
     if overrides:
-        scope_overrides.update(overrides)
+        scope_overrides.update(normalize_runtime_config(overrides, exclude_crawl_execution=True, json_safe=False))
 
     legacy_scope_permissions = permissions_from_legacy_public_flags(scope_overrides)
     if legacy_scope_permissions:
@@ -758,28 +1168,60 @@ def get_config(
     archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
     config_data.update(archivebox_scope_overrides)
 
+    explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
-        plugin_schemas = {
-            plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
-        }
+        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
-        plugin_user_config = _plugin_user_config(scope_overrides)
+        crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
+        # A frozen crawl-level PLUGINS selector is the exact extractor set for
+        # this crawl. Env/file *_ENABLED values are ambient defaults and must
+        # not be re-applied after validation, or a later process env can change
+        # which hooks an already-queued crawl runs.
+        if not crawl_selected_plugins:
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(os.environ))
+        explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(scope_overrides))
+        plugin_user_config = _plugin_user_config(
+            {
+                **_plugin_input_config(os.environ),
+                **scope_overrides,
+            },
+        )
         if not crawl_config_base:
-            plugin_user_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **plugin_user_config}
+            file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(file_config))
+            plugin_user_config = {
+                **_plugin_user_config(_plugin_input_config(file_config)),
+                **plugin_user_config,
+            }
         plugin_sections = resolve_plugin_configs(
             plugin_schemas,
             global_config=plugin_global_config,
             user_config=plugin_user_config,
+            environ={},
         )
         for plugin_config in plugin_sections.values():
-            config_data.update(plugin_config)
+            for key, value in plugin_config.items():
+                if key in ArchiveBoxBaseConfig.model_fields and key not in archivebox_scope_overrides and key not in base_config_payload:
+                    continue
+                config_data[key] = value
         if base_config_payload:
-            config_data.update({key: value for key, value in base_config_payload.items() if key in _archivebox_config_input_names()})
+            config_data.update(
+                {
+                    key: value
+                    for key, value in normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False).items()
+                    if key in _archivebox_config_input_names()
+                },
+            )
         if crawl_config_base:
-            config_data.update(dict(crawl.config or {}))
+            config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
-
-    config_data["ABX_RUNTIME"] = "archivebox"
+        if crawl_selected_plugins:
+            # Crawl PLUGINS is a frozen selector. Drop ambient enable flags from
+            # env/file/defaults before model validation so the selected plugin
+            # set is derived from the crawl, not from the worker process that
+            # happens to resume it.
+            for enabled_key in _plugin_enabled_config_keys().values():
+                config_data.pop(enabled_key, None)
 
     # Decode JSON-encoded complex values (dict/list fields) that came from
     # string-only sources before validation. ``IniConfigSettingsSource`` does
@@ -805,14 +1247,21 @@ def get_config(
             )
 
     config = ArchiveBoxConfig.model_validate(config_data)
+    if config.SERVER_SECURITY_MODE == "auto":
+        base_host = (urlparse(config.BASE_URL if "://" in config.BASE_URL else f"//{config.BASE_URL}").hostname or "").lower()
+        if base_host.endswith(".localhost"):
+            config = config.model_copy(update={"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay"})
+    for key in explicit_plugin_enabled_keys:
+        if key in config_data:
+            setattr(config, key, config_data[key])
+    if config.PLUGINS:
+        config._derive_plugin_enabled_config(respect_current_enabled=True)
     if redact_sensitive:
         for key in type(config).model_fields:
-            value = getattr(config, key, None)
+            value = config[key]
             if is_sensitive_config_key(key) and value not in (None, ""):
                 setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
-    os.environ["LIB_DIR"] = str(config.LIB_DIR)
-    os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
-    os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
+    os.environ["ABXPKG_LIB_DIR"] = str(config.ABXPKG_LIB_DIR)
     archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
     if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
         config.warn_if_invalid()

archivebox/config/configset.py

@@ -43,7 +43,7 @@ def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
     config_path = Path(config_path_str)
     try:
         mtime = config_path.stat().st_mtime
-    except FileNotFoundError:
+    except (FileNotFoundError, PermissionError):
         return {}
     cache_key = (str(config_path), mtime)
     cached = _INI_CACHE.get(cache_key)

archivebox/config/constants.py

@@ -36,8 +36,8 @@ from .permissions import (
     IN_DOCKER,
     RUNNING_AS_UID,
     RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
     ARCHIVEBOX_USER,
     ARCHIVEBOX_GROUP,
 )
@@ -66,8 +66,8 @@ class ConstantsDict:
     ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
     RUNNING_AS_UID: int = RUNNING_AS_UID
     RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
     IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix
 
     # Source code dirs
@@ -113,12 +113,10 @@ class ConstantsDict:
     TMP_DIR_NAME: str = "tmp"
     DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323
 
-    LIB_DIR_NAME: str = "lib"
-    DEFAULT_LIB_DIR: Path = _env_path(
-        "LIB_DIR",
-        user_config_path("abx") / LIB_DIR_NAME,
+    DEFAULT_ABXPKG_LIB_DIR: Path = _env_path(
+        "ABXPKG_LIB_DIR",
+        user_config_path("abx") / "lib",
     )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
         (
@@ -241,7 +239,7 @@ class ConstantsDict:
             SOURCES_DIR_NAME,
             LOGS_DIR_NAME,
             CACHE_DIR_NAME,
-            LIB_DIR_NAME,
+            "lib",
             TMP_DIR_NAME,
             PERSONAS_DIR_NAME,
             CUSTOM_TEMPLATES_DIR_NAME,
@@ -295,5 +293,3 @@ CONSTANTS = ConstantsDict
 CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")})
 
 # add all key: values to globals() for easier importing, e.g.:
-# from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...
-# globals().update(CONSTANTS)

archivebox/config/django.py

@@ -2,19 +2,17 @@ __package__ = "archivebox.config"
 
 import os
 import sys
-import subprocess
-
-from datetime import datetime, timezone
-
-from rich.console import Console
+from datetime import UTC, datetime
 
 import django
 import django.db
+from django.core.exceptions import ImproperlyConfigured
+from rich.console import Console
 
 from archivebox.misc import logging
 
-from .constants import CONSTANTS
 from .common import get_config
+from .constants import CONSTANTS
 
 CONFIG = get_config()
 
@@ -31,32 +29,40 @@ logging.CONSOLE = CONSOLE
 DJANGO_SET_UP = False
 
 
-def setup_django(check_db=False, in_memory_db=False) -> None:
+def setup_django(check_db=False) -> None:
     from rich.panel import Panel
 
+    from archivebox.misc.checks import check_not_inside_source_dir
+
     global DJANGO_SET_UP
 
     if DJANGO_SET_UP:
-        # raise Exception('django is already set up!')
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
+    check_not_inside_source_dir(CONSTANTS.DATA_DIR)
+
+    # SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
+    # ArchiveBox file-mode policy before any DB connection can create the file,
+    # otherwise a permissive parent umask can expose a just-created DB until a
+    # later chmod runs.
+    os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
+
     # Third-party patches are only needed once Django/apps are about to load.
     # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
     # cost for cheap CLI startup paths like `archivebox <cmd> --help`.
     import archivebox.misc.monkey_patches  # noqa: F401
-
-    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
+    from archivebox.config.permissions import ARCHIVEBOX_GROUP, ARCHIVEBOX_USER, IS_ROOT, SudoPermission
 
     # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
     if IS_ROOT and ARCHIVEBOX_USER != 0:
         with SudoPermission(uid=0):
             # running as root is a special case where it's ok to be a bit slower
             # make sure data dir is always owned by the correct user
-            subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(CONSTANTS.DATA_DIR)], stderr=subprocess.DEVNULL)
+            os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
             if CONSTANTS.DATA_DIR.exists():
                 for child in CONSTANTS.DATA_DIR.iterdir():
-                    subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(child)], stderr=subprocess.DEVNULL)
+                    os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
 
     # Suppress the "database access during app initialization" warning
     # This warning can be triggered during django.setup() but is safe to ignore
@@ -72,40 +78,30 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
     try:
         from django.core.management import call_command
 
-        if in_memory_db:
-            raise Exception("dont use this anymore")
-
-            # some commands dont store a long-lived sqlite3 db file on disk.
-            # in those cases we create a temporary in-memory db and run the migrations
-            # immediately to get a usable in-memory-database at startup
-            os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
+        # Initialize the configured file-based database without running
+        # migrations automatically; `archivebox init` owns migrations.
+        try:
             django.setup()
+        # User config and import errors are reported through one CLI message.
+        except (ImproperlyConfigured, django.db.Error, ImportError, OSError, RuntimeError, ValueError) as e:
+            is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version"))
+            if not is_using_meta_cmd:
+                # show error message to user only if they're not running a meta command / just trying to get help
+                STDERR.print()
+                STDERR.print(
+                    Panel(
+                        f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n",
+                        title="\n\n[red][X] Error while trying to load database![/red]",
+                        subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]",
+                        expand=False,
+                        style="bold red",
+                    ),
+                )
+                STDERR.print()
+                import traceback
 
-            call_command("migrate", interactive=False, verbosity=0)
-        else:
-            # Otherwise use default sqlite3 file-based database and initialize django
-            # without running migrations automatically (user runs them manually by calling init)
-            try:
-                django.setup()
-            except Exception as e:
-                is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version"))
-                if not is_using_meta_cmd:
-                    # show error message to user only if they're not running a meta command / just trying to get help
-                    STDERR.print()
-                    STDERR.print(
-                        Panel(
-                            f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n",
-                            title="\n\n[red][X] Error while trying to load database![/red]",
-                            subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]",
-                            expand=False,
-                            style="bold red",
-                        ),
-                    )
-                    STDERR.print()
-                    import traceback
-
-                    traceback.print_exc()
-                return
+                traceback.print_exc()
+            return
 
         from archivebox.core.settings_logging import ERROR_LOG as DEFAULT_ERROR_LOG
 
@@ -113,13 +109,13 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
         error_log = DEFAULT_ERROR_LOG
         with open(error_log, "a", encoding="utf-8") as f:
             command = " ".join(sys.argv)
-            ts = datetime.now(timezone.utc).strftime("%Y-%m-%d__%H:%M:%S")
+            ts = datetime.now(UTC).strftime("%Y-%m-%d__%H:%M:%S")
             config = get_config()
             f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={config.IN_DOCKER} IS_TTY={config.IS_TTY}\n")
 
         if check_db:
             # make sure the data dir is owned by a non-root user
-            if CONSTANTS.DATA_DIR.stat().st_uid == 0:
+            if CONSTANTS.DATA_DIR.stat().st_uid == 0 and not (IS_ROOT and ARCHIVEBOX_USER == 0):
                 STDERR.print("[red][X] Error: ArchiveBox DATA_DIR cannot be owned by root![/red]")
                 STDERR.print(f"    {CONSTANTS.DATA_DIR}")
                 STDERR.print()
@@ -145,23 +141,14 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
             for conn in connections.all():
                 conn.close_if_unusable_or_obsolete()
 
-            sql_index_path = CONSTANTS.DATABASE_FILE
-            assert os.access(sql_index_path, os.F_OK), (
-                f"No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)"
+            from archivebox.misc.db import database_display_location, database_exists
+
+            assert database_exists(), (
+                f"No database {database_display_location()} found for: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)"
             )
 
-            # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
-            # if settings.DEBUG_LOGFIRE:
-            #     from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
-            #     SQLite3Instrumentor().instrument()
-
-            #     import logfire
-
-            #     logfire.configure()
-            #     logfire.instrument_django(is_sql_commentor_enabled=True)
-            #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
-
     except KeyboardInterrupt:
+        DJANGO_SET_UP = False
         raise
 
     DJANGO_SET_UP = True

archivebox/config/paths.py

@@ -5,7 +5,6 @@ import socket
 import hashlib
 import tempfile
 import platform
-import subprocess
 from pathlib import Path
 from functools import cache
 from datetime import datetime
@@ -19,7 +18,9 @@ if TYPE_CHECKING:
 #############################################################################################
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -29,8 +30,8 @@ def _env_path(key: str, default: Path) -> Path:
     return path.resolve()
 
 
-ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive")  # archivebox snapshot data dir
-USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users")  # archivebox user-scoped crawl/snapshot data dir
+ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir
+USERS_DIR: Path = ARCHIVE_DIR / "users"  # archivebox user-scoped crawl/snapshot data dir
 
 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
 
@@ -57,9 +58,11 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
     collection_id = hashlib.sha256(f"{machine_id}:{collection_path}@{creation_date}".encode()).hexdigest()[:8]
 
     try:
-        # only persist collection_id file if we already have an index.sqlite3 file present
+        # only persist collection_id file if this dir already looks like a real collection
+        # (has an index.sqlite3, or an ArchiveBox.conf when the DB lives in postgres),
         # otherwise we might be running in a directory that is not a collection, no point creating cruft files
-        collection_is_active = os.path.isfile(DATABASE_FILE) and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
+        collection_marker = os.path.isfile(DATABASE_FILE) or os.path.isfile(DATA_DIR / "ArchiveBox.conf")
+        collection_is_active = collection_marker and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
         if collection_is_active or force_create:
             collection_id_file.write_text(collection_id)
 
@@ -67,9 +70,9 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
             if IS_ROOT:
                 with SudoPermission(uid=0):
                     if ARCHIVEBOX_USER == 0:
-                        subprocess.run(["chmod", "777", str(collection_id_file)])
+                        collection_id_file.chmod(0o777)
                     else:
-                        subprocess.run(["chown", str(ARCHIVEBOX_USER), str(collection_id_file)])
+                        os.chown(collection_id_file, ARCHIVEBOX_USER, -1)
     except (OSError, FileNotFoundError, PermissionError):
         pass
     return collection_id
@@ -120,14 +123,14 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
     try:
         with SudoPermission(uid=uid, fallback=fallback):
             test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
             test_file.unlink()
             return True
     except (OSError, PermissionError):
         if chown:
             # try fixing it using sudo permissions
             with SudoPermission(uid=uid, fallback=fallback):
-                subprocess.run(["chown", f"{uid}:{gid}", str(dir_path)], stderr=subprocess.DEVNULL)
+                os.chown(dir_path, uid, gid)
             return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False)
     return False
 
@@ -176,33 +179,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
 
 
 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates
 
 
 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
     from archivebox.misc.checks import check_tmp_dir
 
     config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
     fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
         try:
             create_and_chown_dir(candidate)
         except Exception:
@@ -231,7 +246,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
         return fallback_candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
 
 
 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -240,8 +255,8 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
 
     config = config or get_config(**config_kwargs)
 
-    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
-    CANDIDATES = [config.LIB_DIR]
+    # ABXPKG_LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.ABXPKG_LIB_DIR]
 
     for candidate in CANDIDATES:
         try:
@@ -249,12 +264,30 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
         except Exception:
             pass
         if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and config.LIB_DIR != candidate:
-                os.environ["LIB_DIR"] = str(candidate)
+            if autofix and config.ABXPKG_LIB_DIR != candidate:
+                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
             return candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")
+
+
+def _sql_index_location() -> dict:
+    from archivebox.misc.db import database_display_location, database_exists, is_postgres
+
+    if is_postgres():
+        return {
+            "path": database_display_location(),
+            "enabled": True,
+            "is_valid": database_exists(),
+            "is_mount": False,
+        }
+    return {
+        "path": DATABASE_FILE.resolve(),
+        "enabled": True,
+        "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
+        "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
+    }
 
 
 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -283,27 +316,22 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
                 and os.access(CONSTANTS.CONFIG_FILE, os.W_OK),
             },
-            "SQL_INDEX": {
-                "path": DATABASE_FILE.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
-                "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
-            },
+            "SQL_INDEX": _sql_index_location(),
             "ARCHIVE_DIR": {
-                "path": config.ARCHIVE_DIR.resolve(),
+                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(config.ARCHIVE_DIR)
-                and os.access(config.ARCHIVE_DIR, os.R_OK)
-                and os.access(config.ARCHIVE_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
+                "is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
             },
             "USERS_DIR": {
-                "path": config.USERS_DIR.resolve(),
-                "enabled": os.path.isdir(config.USERS_DIR),
-                "is_valid": os.path.isdir(config.USERS_DIR)
-                and os.access(config.USERS_DIR, os.R_OK)
-                and os.access(config.USERS_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.USERS_DIR.resolve()),
+                "path": CONSTANTS.USERS_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.USERS_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
+                and os.access(CONSTANTS.USERS_DIR, os.R_OK)
+                and os.access(CONSTANTS.USERS_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
             },
             "SOURCES_DIR": {
                 "path": CONSTANTS.SOURCES_DIR.resolve(),
@@ -329,13 +357,11 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
             "TMP_DIR": {
                 "path": tmp_dir.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
-            # "CACHE_DIR": {
-            #     "path": CACHE_DIR.resolve(),
-            #     "enabled": True,
-            #     "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK),                        # read + write
-            # },
         },
     )
 
@@ -347,11 +373,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 
     config = config or get_config(**config_kwargs)
     try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
     except Exception:
-        lib_dir = config.LIB_DIR
-
-    lib_bin_dir = lib_dir / "bin"
+        lib_dir = config.ABXPKG_LIB_DIR
 
     return AttrDict(
         {
@@ -366,26 +390,19 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),  # read + list
             },
             "CUSTOM_TEMPLATES_DIR": {
-                "path": config.CUSTOM_TEMPLATES_DIR.resolve(),
-                "enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
-                "is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
+                "path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
             },
             "USER_PLUGINS_DIR": {
                 "path": CONSTANTS.USER_PLUGINS_DIR.resolve(),
                 "enabled": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
                 "is_valid": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),  # read
             },
-            "LIB_DIR": {
+            "ABXPKG_LIB_DIR": {
                 "path": lib_dir.resolve(),
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
             },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
         },
     )

archivebox/config/permissions.py

@@ -1,22 +1,54 @@
 __package__ = "archivebox.config"
 
 import os
-import pwd
-import sys
-import socket
 import platform
+import pwd
+import socket
+import stat
+import subprocess
+import sys
+from contextlib import contextmanager, suppress
+from pathlib import Path
 from typing import cast
 
 from rich import print
 
-from pathlib import Path
-from contextlib import contextmanager
-
 #############################################################################################
 
+
+def is_root_identity(running_uid: int, effective_uid: int) -> bool:
+    return running_uid == 0 or effective_uid == 0
+
+
+def select_archivebox_user(
+    *,
+    running_uid: int,
+    running_gid: int,
+    effective_uid: int,
+    effective_gid: int,
+    sudo_uid: int,
+    sudo_gid: int,
+    data_dir_uid: int,
+    data_dir_gid: int,
+    account_uid: int | None,
+    account_gid: int | None,
+    data_dir_owner_gid: int | None = None,
+    data_dir_owner_exists: bool = True,
+) -> tuple[int, int]:
+    if running_uid == 0:
+        if data_dir_uid != 0 and data_dir_owner_exists:
+            owner_gid = data_dir_owner_gid if data_dir_gid == 0 and data_dir_owner_gid is not None else data_dir_gid
+            return data_dir_uid, owner_gid
+        if account_uid is not None and account_gid is not None:
+            return account_uid, account_gid
+
+    return (
+        effective_uid or running_uid or sudo_uid,
+        effective_gid or running_gid or sudo_gid,
+    )
+
+
 DATA_DIR = Path(os.getcwd())
-if os.environ.get("DATA_DIR") and Path(os.environ["DATA_DIR"]).resolve() != DATA_DIR.resolve():
-    raise SystemExit(f"[X] DATA_DIR={os.environ['DATA_DIR']} must equal cwd={DATA_DIR}; cd into the data dir before running archivebox")
 
 try:
     DATA_DIR_STAT = DATA_DIR.stat()
@@ -26,47 +58,100 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+try:
+    DATA_DIR_OWNER_GID = pwd.getpwuid(DATA_DIR_UID).pw_gid
+    DATA_DIR_OWNER_EXISTS = True
+except KeyError:
+    DATA_DIR_OWNER_GID = None
+    DATA_DIR_OWNER_EXISTS = False
+
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
 EGID = os.getegid()
-SUDO_UID = int(os.environ.get("SUDO_UID", 0))
-SUDO_GID = int(os.environ.get("SUDO_GID", 0))
+SUDO_UID = int(os.environ.get("SUDO_UID", "0"))
+SUDO_GID = int(os.environ.get("SUDO_GID", "0"))
 USER: str = Path("~").expanduser().resolve().name
 HOSTNAME: str = cast(str, max([socket.gethostname(), platform.node()], key=len))
 
-IS_ROOT = RUNNING_AS_UID == 0
-IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
-# IN_DOCKER_COMPOSE =  # TODO: figure out a way to detect if running in docker compose
-
+IS_ROOT = is_root_identity(RUNNING_AS_UID, EUID)
+IN_DOCKER = os.environ.get("IN_DOCKER", "") in ("1", "true", "True", "TRUE", "yes")
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
+
+
+def get_or_create_archivebox_account():
     try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+        account = pwd.getpwnam("archivebox")
+    except KeyError:
+        account = None
+
+    if account is None and (RUNNING_AS_UID != 0 or platform.system() != "Linux"):
+        return None
+
+    if account is None:
+        useradd = "/usr/sbin/useradd" if Path("/usr/sbin/useradd").exists() else "useradd"
+        command = [
+            useradd,
+            "--system",
+            "--create-home",
+            "--home-dir",
+            "/var/lib/archivebox",
+            "--shell",
+            "/bin/bash",
+            "archivebox",
+        ]
+        try:
+            subprocess.run(command, check=True)
+        except (OSError, subprocess.CalledProcessError) as err:
+            # Another concurrently starting process may have created it first.
+            try:
+                account = pwd.getpwnam("archivebox")
+            except KeyError:
+                raise RuntimeError(f"Failed to create the archivebox system user with: {' '.join(command)}") from err
+        else:
+            account = pwd.getpwnam("archivebox")
+
+    if RUNNING_AS_UID == 0 and platform.system() == "Linux":
+        # Package removal may preserve the account but remove its home. Repair
+        # only this directory entry; never recursively chown existing data.
+        account_home = Path(account.pw_dir)
+        try:
+            account_home.mkdir(parents=True, exist_ok=True)
+            home_stat = account_home.stat()
+            if (home_stat.st_uid, home_stat.st_gid) != (account.pw_uid, account.pw_gid):
+                os.chown(account_home, account.pw_uid, account.pw_gid)
+        except OSError as err:
+            raise RuntimeError(f"Failed to prepare archivebox account home: {account_home}") from err
+
+    return account
 
 
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
+ARCHIVEBOX_ACCOUNT = get_or_create_archivebox_account()
 
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+ARCHIVEBOX_USER, ARCHIVEBOX_GROUP = select_archivebox_user(
+    running_uid=RUNNING_AS_UID,
+    running_gid=RUNNING_AS_GID,
+    effective_uid=EUID,
+    effective_gid=EGID,
+    sudo_uid=SUDO_UID,
+    sudo_gid=SUDO_GID,
+    data_dir_uid=DATA_DIR_UID,
+    data_dir_gid=DATA_DIR_GID,
+    data_dir_owner_gid=DATA_DIR_OWNER_GID,
+    account_uid=ARCHIVEBOX_ACCOUNT.pw_uid if ARCHIVEBOX_ACCOUNT is not None else None,
+    account_gid=ARCHIVEBOX_ACCOUNT.pw_gid if ARCHIVEBOX_ACCOUNT is not None else None,
+    data_dir_owner_exists=DATA_DIR_OWNER_EXISTS,
+)
 if not USER:
     try:
         # alternative method 1 to get username
         USER = pwd.getpwuid(ARCHIVEBOX_USER).pw_name
-    except Exception:
-        pass
+    except (KeyError, OSError):
+        USER = ""
 
 if not USER:
     try:
@@ -74,42 +159,172 @@ if not USER:
         import getpass
 
         USER = getpass.getuser()
-    except Exception:
-        pass
+    except OSError:
+        USER = ""
 
 if not USER:
     try:
         # alternative method 3 to get username
         USER = os.getlogin() or "archivebox"
-    except Exception:
+    except OSError:
         USER = "archivebox"
 
 ARCHIVEBOX_USER_EXISTS = False
 try:
     pwd.getpwuid(ARCHIVEBOX_USER)
     ARCHIVEBOX_USER_EXISTS = True
-except Exception:
+except KeyError:
     ARCHIVEBOX_USER_EXISTS = False
 
 
+ROOT_HANDOFF_NAMES = (
+    ".archivebox_id",
+    "ArchiveBox.conf",
+    "index.sqlite3",
+    "index.sqlite3-shm",
+    "index.sqlite3-wal",
+    "archive",
+    "cache",
+    "custom_plugins",
+    "custom_templates",
+    "lib",
+    "logs",
+    "personas",
+    "sonic",
+    "sources",
+    "tmp",
+    "users",
+)
+
+
+def root_data_dir_handoff_paths(data_dir: Path, argv: list[str]) -> tuple[Path, ...]:
+    """Return bounded top-level paths safe to hand from root to archivebox."""
+
+    data_dir = data_dir.resolve()
+    if data_dir == Path("/"):
+        return ()
+
+    try:
+        children = tuple(data_dir.iterdir())
+    except (FileNotFoundError, PermissionError):
+        return ()
+
+    is_init = any(arg in ("init", "install", "--init", "--quick-init") for arg in argv[1:])
+    collection_exists = any((data_dir / marker).exists() for marker in (".archivebox_id", "ArchiveBox.conf", "index.sqlite3"))
+    if not collection_exists and not (is_init and not children):
+        return ()
+
+    top_level_paths = tuple(data_dir / name for name in ROOT_HANDOFF_NAMES if (data_dir / name).exists())
+    log_files = tuple((data_dir / "logs").glob("*.log"))
+    return (data_dir, *top_level_paths, *log_files)
+
+
+def root_should_handoff_data_dir(
+    *,
+    is_root: bool,
+    data_dir_uid: int,
+    account_uid: int | None,
+    data_dir_owner_exists: bool = True,
+) -> bool:
+    """Allow bounded handoff only for root or archivebox-owned collection roots."""
+
+    return is_root and account_uid is not None and (not data_dir_owner_exists or data_dir_uid in (0, account_uid))
+
+
+def root_parent_can_grant_group_traversal(*, parent_uid: int, parent_gid: int, parent_mode: int, account_gid: int) -> bool:
+    """Return whether adding archivebox traversal preserves existing parent access."""
+
+    if parent_uid != 0 or parent_mode & stat.S_IXOTH:
+        return False
+    if parent_gid == account_gid:
+        return not bool(parent_mode & stat.S_IXGRP)
+    return not bool(parent_mode & stat.S_IRWXG)
+
+
+def grant_archivebox_group_traversal(path: Path) -> None:
+    """Let the archivebox account traverse private root-owned parents."""
+
+    if not IS_ROOT or ARCHIVEBOX_ACCOUNT is None:
+        return
+
+    for parent in path.resolve().parents:
+        if parent == Path("/"):
+            continue
+        parent_stat = parent.stat(follow_symlinks=False)
+        if root_parent_can_grant_group_traversal(
+            parent_uid=parent_stat.st_uid,
+            parent_gid=parent_stat.st_gid,
+            parent_mode=parent_stat.st_mode,
+            account_gid=ARCHIVEBOX_ACCOUNT.pw_gid,
+        ):
+            os.chown(parent, -1, ARCHIVEBOX_ACCOUNT.pw_gid, follow_symlinks=False)
+            os.chmod(parent, stat.S_IMODE(parent_stat.st_mode) | stat.S_IXGRP, follow_symlinks=False)
+
+
+def handoff_root_owned_data_dir() -> None:
+    account_uid = ARCHIVEBOX_ACCOUNT.pw_uid if ARCHIVEBOX_ACCOUNT is not None else None
+    if not root_should_handoff_data_dir(
+        is_root=IS_ROOT,
+        data_dir_uid=DATA_DIR_UID,
+        account_uid=account_uid,
+        data_dir_owner_exists=DATA_DIR_OWNER_EXISTS,
+    ):
+        return
+
+    handoff_paths = root_data_dir_handoff_paths(DATA_DIR, sys.argv)
+    if not handoff_paths:
+        return
+
+    # Never walk collection contents. This only grants execute-only traversal
+    # on private parents such as /root so the handed-off data remains reachable.
+    grant_archivebox_group_traversal(DATA_DIR)
+
+    for path in handoff_paths:
+        try:
+            os.chown(path, ARCHIVEBOX_ACCOUNT.pw_uid, ARCHIVEBOX_ACCOUNT.pw_gid, follow_symlinks=False)
+        except (FileNotFoundError, PermissionError):
+            pass
+
+
 #############################################################################################
 
 
 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # always run archivebox as the user that owns the data dir, never as root
-    if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
-        if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
+    # Root-owned uv tools commonly live below /root. Keep the installed package
+    # importable after dropping EUID without changing ownership of the tool env.
+    grant_archivebox_group_traversal(Path(__file__))
+    handoff_root_owned_data_dir()
+
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
+    if os.geteuid() == 0 and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
+        pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
+        supplemental_gid = ARCHIVEBOX_ACCOUNT.pw_gid if ARCHIVEBOX_ACCOUNT is not None else ARCHIVEBOX_GROUP
+        os.initgroups(pw_record.pw_name, supplemental_gid)
+        if os.getegid() != ARCHIVEBOX_GROUP:
+            os.setegid(ARCHIVEBOX_GROUP)
+        if os.geteuid() != ARCHIVEBOX_USER:
             os.seteuid(ARCHIVEBOX_USER)
 
-            # update environment variables so that subprocesses dont try to write to /root
-            pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
-            os.environ["HOME"] = pw_record.pw_dir
-            os.environ["LOGNAME"] = pw_record.pw_name
-            os.environ["USER"] = pw_record.pw_name
+        # update environment variables so that subprocesses dont try to write to /root
+        os.environ["HOME"] = pw_record.pw_dir
+        os.environ["LOGNAME"] = pw_record.pw_name
+        os.environ["USER"] = pw_record.pw_name
+        os.environ["XDG_CACHE_HOME"] = str(Path(pw_record.pw_dir) / ".cache")
+        os.environ["XDG_CONFIG_HOME"] = str(Path(pw_record.pw_dir) / ".config")
+        os.environ["XDG_DATA_HOME"] = str(Path(pw_record.pw_dir) / ".local" / "share")
+        os.environ.pop("XDG_RUNTIME_DIR", None)
+        os.environ.pop("ABXBUS_MULTIPROCESS_SEMAPHORE_DIR", None)
+
+        semaphore_dir = Path(pw_record.pw_dir) / ".cache" / "abxbus" / "semaphores"
+        os.environ["ABXBUS_MULTIPROCESS_SEMAPHORE_DIR"] = str(semaphore_dir)
+
+        with suppress(ImportError):
+            from abxbus import retry as abxbus_retry
+
+            abxbus_retry.MULTIPROCESS_SEMAPHORE_DIR = semaphore_dir
 
     if ARCHIVEBOX_USER == 0 or not ARCHIVEBOX_USER_EXISTS:
         print(

archivebox/config/version.py

@@ -6,6 +6,7 @@ import importlib.metadata
 from pathlib import Path
 from functools import cache
 from datetime import datetime
+import re
 
 #############################################################################################
 
@@ -35,22 +36,76 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
         # building docs, pyproject.toml is not available
         pass
 
-    # raise Exception('Failed to detect installed archivebox version!')
     return "dev"
 
 
 @cache
 def get_COMMIT_HASH() -> str | None:
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH",):
+        env_commit_hash = os.environ.get(env_var, "").strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
+            return env_commit_hash
+
+    if IN_DOCKER:
+        try:
+            version_text = Path("/VERSION.txt").read_text()
+            matches = re.findall(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
+            if matches:
+                return matches[-1]
+        except Exception:
+            pass
+
     try:
-        git_dir = PACKAGE_DIR.parent / ".git"
-        ref = (git_dir / "HEAD").read_text().strip().split(" ")[-1]
-        commit_hash = git_dir.joinpath(ref).read_text().strip()
-        return commit_hash
+        packaged_commit_hash = (PACKAGE_DIR / "COMMIT_SHA").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", packaged_commit_hash):
+            return packaged_commit_hash
     except Exception:
         pass
 
+    def _read_git_file(git_dir: Path, ref: str) -> str | None:
+        try:
+            return git_dir.joinpath(ref).read_text().strip()
+        except Exception:
+            pass
+
+        try:
+            packed_refs = git_dir.joinpath("packed-refs").read_text().splitlines()
+        except Exception:
+            return None
+
+        for line in packed_refs:
+            if line.startswith("#") or line.startswith("^") or not line.strip():
+                continue
+            commit_hash, packed_ref = line.split(" ", 1)
+            if packed_ref == ref:
+                return commit_hash.strip()
+
+        return None
+
     try:
-        return list((PACKAGE_DIR.parent / ".git/refs/heads/").glob("*"))[0].read_text().strip()
+        try:
+            pyproject_text = (PACKAGE_DIR.parent / "pyproject.toml").read_text()
+        except FileNotFoundError:
+            pyproject_text = ""
+        if not re.search(r'^name = "archivebox"$', pyproject_text, re.MULTILINE):
+            return None
+
+        git_dir = PACKAGE_DIR.parent / ".git"
+        if git_dir.is_file():
+            gitdir_line = git_dir.read_text().strip()
+            gitdir_path = gitdir_line.removeprefix("gitdir:").strip()
+            git_dir = Path(gitdir_path)
+            if not git_dir.is_absolute():
+                git_dir = PACKAGE_DIR.parent / git_dir
+
+        head = (git_dir / "HEAD").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", head):
+            return head
+
+        ref = head.removeprefix("ref:").strip()
+        commit_hash = _read_git_file(git_dir, ref)
+        if commit_hash:
+            return commit_hash
     except Exception:
         pass
 
@@ -59,65 +114,13 @@ def get_COMMIT_HASH() -> str | None:
 
 @cache
 def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        try:
-            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
-            docker_build_end_time = Path("/VERSION.txt").read_text().rsplit("BUILD_END_TIME=")[-1].split("\n", 1)[0]
-            return docker_build_end_time
-        except Exception:
-            pass
+    for env_var in ("ARCHIVEBOX_BUILD_TIME", "BUILD_TIME"):
+        build_time = os.environ.get(env_var, "").strip()
+        if build_time:
+            return build_time
 
     src_last_modified_unix_timestamp = (PACKAGE_DIR / "README.md").stat().st_mtime
     return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")
 
 
-# def get_versions_available_on_github(config):
-#     """
-#     returns a dictionary containing the ArchiveBox GitHub release info for
-#     the recommended upgrade version and the currently installed version
-#     """
-
-#     # we only want to perform the (relatively expensive) check for new versions
-#     # when its most relevant, e.g. when the user runs a long-running command
-#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
-#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
-#     if subcommand_run_by_user not in long_running_commands:
-#         return None
-
-#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
-#     response = requests.get(github_releases_api)
-#     if response.status_code != 200:
-#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
-#         return None
-#     all_releases = response.json()
-
-#     installed_version = parse_version_string(config['VERSION'])
-
-#     # find current version or nearest older version (to link to)
-#     current_version = None
-#     for idx, release in enumerate(all_releases):
-#         release_version = parse_version_string(release['tag_name'])
-#         if release_version <= installed_version:
-#             current_version = release
-#             break
-
-#     current_version = current_version or all_releases[-1]
-
-#     # recommended version is whatever comes after current_version in the release list
-#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
-#     try:
-#         recommended_version = all_releases[idx+1]
-#     except IndexError:
-#         recommended_version = None
-
-#     return {'recommended_version': recommended_version, 'current_version': current_version}
-
-# def can_upgrade(config):
-#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
-#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
-#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
-#         return recommended_version > current_version
-#     return False
-
-
 VERSION: str = detect_installed_version()

archivebox/config/views.py

@@ -1,30 +1,36 @@
 __package__ = "archivebox.config"
 
-import os
 import inspect
+import os
 from pathlib import Path
 from typing import Any
 from urllib.parse import quote, urlencode
+
+from admin_data_views.typing import ItemContext, SectionData, TableContext
+from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
 from django.http import HttpRequest
 from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
-from admin_data_views.typing import TableContext, ItemContext, SectionData
-from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
-
 from archivebox.config import CONSTANTS
+from archivebox.machine.models import Binary
 from archivebox.misc.util import parse_date
 
-from archivebox.machine.models import Binary
-
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
+LOG_DETAIL_TAIL_BYTES = 100_000
+
+
+def _read_text_tail(path: Path, max_bytes: int = LOG_DETAIL_TAIL_BYTES) -> str:
+    with path.open("rb") as log_file:
+        size = path.stat().st_size
+        log_file.seek(max(size - max_bytes, 0))
+        return log_file.read(max_bytes).decode("utf-8", errors="replace")
 
 
 def is_superuser(request: HttpRequest) -> bool:
-    return bool(getattr(request.user, "is_superuser", False))
+    return bool(request.user.is_superuser)
 
 
 def format_parsed_datetime(value: object) -> str:
@@ -32,10 +38,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
     return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
 
@@ -44,7 +46,7 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
     if binary is None or not binary.id:
         return None
 
-    base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
+    base_url = binary.admin_change_url
     changelist_filters = urlencode({"q": name})
     return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
 
@@ -92,7 +94,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
             return f" {obj}"
 
     elif isinstance(obj, (int, float, bool)):
-        return f" {str(obj)}"
+        return f" {obj!s}"
 
     elif callable(obj):
         source = (
@@ -103,7 +105,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
         return f" {indent_str}  " + source.replace("\n", f"\n{indent_str}  ")
 
     else:
-        return f" {str(obj)}"
+        return f" {obj!s}"
 
 
 def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
@@ -158,7 +160,7 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and getattr(binary, "is_valid", getattr(binary, "abspath", None)))
+        binary_is_valid = bool(binary and binary.is_valid)
 
         rows["Binary Name"].append(ItemLink(name, key=name))
 
@@ -185,9 +187,9 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         "youtube-dl": "yt-dlp",
     }.get(key, key)
     db_binary = get_db_binaries_by_name().get(key)
-    binary_is_valid = bool(db_binary and getattr(db_binary, "is_valid", getattr(db_binary, "abspath", None)))
+    binary_is_valid = bool(db_binary and db_binary.is_valid)
     if binary_is_valid:
-        binary_data = db_binary.to_json() if hasattr(db_binary, "to_json") else db_binary.__dict__
+        binary_data = db_binary.to_json()
         section: SectionData = {
             "name": key,
             "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),
@@ -305,7 +307,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 pid_to_process_id[row.pid] = str(row.id)
                 pid_to_process_type[row.pid] = row.process_type
 
-        except Exception:
+        except (ImportError, RuntimeError, TypeError, ValueError):
             pass
 
     def _pid_cell(pid_value: int | None, uptime_str: str = ""):
@@ -346,7 +348,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
         hours, remainder = divmod(seconds, 3600)
         minutes, secs = divmod(remainder, 60)
         supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
-    except Exception:
+    except (ImportError, RuntimeError, TypeError, ValueError):
         try:
             from archivebox.machine.models import Machine, Process
 
@@ -365,7 +367,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 hours, remainder = divmod(seconds, 3600)
                 minutes, secs = divmod(remainder, 60)
                 supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
-        except Exception:
+        except (RuntimeError, TypeError, ValueError):
             pass
 
     rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
@@ -420,7 +422,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
 def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
-    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME
+    from archivebox.workers.supervisord_util import CONFIG_FILE_NAME, get_existing_supervisord_process, get_sock_file, get_worker
 
     SOCK_FILE = get_sock_file()
     CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME
@@ -443,8 +445,12 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
     if key == "supervisord":
         relevant_config = CONFIG_FILE.read_text()
-        relevant_logs = str(supervisor.readLog(0, 10_000_000))
-        start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
+        supervisor_log = CONSTANTS.LOGS_DIR / "supervisord.log"
+        with supervisor_log.open("rb") as log_file:
+            startup_logs = log_file.read(LOG_DETAIL_TAIL_BYTES).decode("utf-8", errors="replace")
+        relevant_logs = _read_text_tail(supervisor_log)
+        startup_lines = [line for line in startup_logs.split("\n") if "RPC interface 'supervisor' initialized" in line]
+        start_ts = startup_lines[-1].split(",", 1)[0] if startup_lines else ""
         start_dt = parse_date(start_ts)
         uptime = str(timezone.now() - start_dt).split(".")[0] if start_dt else ""
         supervisor_state = supervisor.getState()
@@ -463,7 +469,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         worker_data = get_worker(supervisor, key)
         proc = worker_data if isinstance(worker_data, dict) else {}
         relevant_config = next((config for config in all_config if config.get("name") == key), {})
-        log_result = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)
+        log_result = supervisor.tailProcessStdoutLog(key, -LOG_DETAIL_TAIL_BYTES, LOG_DETAIL_TAIL_BYTES)
         relevant_logs = str(log_result[0] if isinstance(log_result, tuple) else log_result)
 
     section: SectionData = {
@@ -479,7 +485,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
             "Logfile": str(proc.get("stdout_logfile") or ""),
             "Uptime": str(str(proc.get("description") or "").split("uptime ", 1)[-1]),
             "Config": obj_to_yaml(relevant_config) if isinstance(relevant_config, dict) else str(relevant_config),
-            "Logs": relevant_logs,
+            "Recent Logs (last 100 KB)": relevant_logs,
         },
         "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
     }
@@ -533,9 +539,9 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
 def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
-    log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name][0]
+    log_file = next(logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name)
 
-    log_text = log_file.read_text()
+    log_text = _read_text_tail(log_file)
     log_stat = log_file.stat()
 
     section: SectionData = {
@@ -545,8 +551,8 @@ def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
             "Path": str(log_file),
             "Size": f"{log_stat.st_size // 1000} kb",
             "Last Updated": format_parsed_datetime(log_stat.st_mtime),
-            "Tail": "\n".join(log_text[-10_000:].split("\n")[-20:]),
-            "Full Log": log_text,
+            "Tail": "\n".join(log_text.split("\n")[-20:]),
+            "Recent Log (last 100 KB)": log_text,
         },
     }
 

archivebox/core/admin_archiveresults.py

@@ -4,38 +4,34 @@ import html
 import json
 import os
 import shlex
-from pathlib import Path
-from urllib.parse import quote
 from functools import reduce
 from operator import and_
+from pathlib import Path
+from urllib.parse import quote
 
 from django.contrib import admin
+from django.core.exceptions import ValidationError
 from django.db.models import Min, Prefetch, Q, TextField
 from django.db.models.functions import Cast
+from django.urls import resolve, reverse
+from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
-from django.core.exceptions import ValidationError
-from django.urls import reverse, resolve
-from django.utils import timezone
 from django.utils.text import smart_split
 
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
-from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.plugins.discovery import get_plugin_icon
-from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
 from archivebox.machine.env_util import env_to_shell_exports
-
-
-from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
 
 
 def _get_replay_source_url(result: ArchiveResult) -> str:
     process = result.process_record
-    return str(getattr(process, "url", None) or result.snapshot.url or "")
+    return str((process.url if process else None) or result.snapshot.url or "")
 
 
 def build_abx_dl_display_command(result: ArchiveResult) -> str:
@@ -51,8 +47,8 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:
 
 def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
     display_command = build_abx_dl_display_command(result)
-    process = getattr(result, "process", None)
-    env_items = env_to_shell_exports(getattr(process, "env", None) or {})
+    process = result.process
+    env_items = env_to_shell_exports(process.env if process else {})
     if config is not None:
         result.snapshot._runtime_config = config
     snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
@@ -77,6 +73,12 @@ def get_plugin_admin_url(plugin_name: str) -> str:
     return f"{LIVE_PLUGIN_BASE_URL}builtin.{quote(plugin_name)}/"
 
 
+def get_process_link_label(process) -> str:
+    if process.pid:
+        return str(process.pid)
+    return str(process.id)[-8:]
+
+
 def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
     """Render a nice inline list view of archive results with status, plugin, output, and actions."""
 
@@ -116,15 +118,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         status = result.status or "queued"
         color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6"))
         output_files = result.output_files or {}
-        if isinstance(output_files, dict):
-            output_file_count = len(output_files)
-        elif isinstance(output_files, (list, tuple, set)):
+        if isinstance(output_files, (dict, list, tuple, set)):
             output_file_count = len(output_files)
         elif isinstance(output_files, str):
             try:
                 parsed = json.loads(output_files)
                 output_file_count = len(parsed) if isinstance(parsed, (dict, list, tuple, set)) else 0
-            except Exception:
+            except (TypeError, ValueError):
                 output_file_count = 0
         else:
             output_file_count = 0
@@ -132,31 +132,43 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         # Get plugin icon
         icon = get_plugin_icon(result.plugin)
 
-        # Format timestamp
-        end_time = result.end_ts.strftime("%Y-%m-%d %H:%M:%S") if result.end_ts else "-"
+        # Keep each timestamp component intact while allowing one wrap between them.
+        if result.end_ts:
+            end_time = (
+                f'<span style="white-space: nowrap;">{result.end_ts:%Y-%m-%d}</span>'
+                f'<wbr> <span style="white-space: nowrap;">{result.end_ts:%H:%M:%S}</span>'
+            )
+        else:
+            end_time = "-"
 
         process = result.process_record
         process_display = "-"
         if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
             process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
+                <a href="{process_url}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{process.pid or "-"}</a>
+                   title="View process">{process_label}</a>
             '''
 
         machine_display = "-"
         if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
             machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
+                <a href="{machine_url}"
                    style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{process.machine.hostname}</a>
+                   title="View machine">{machine_label}</a>
             '''
 
         # Truncate output for display
-        full_output = result.output_str_for_display() or "-"
-        output_display = full_output[:60]
-        if len(full_output) > 60:
-            output_display += "..."
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)
 
         display_cmd = build_abx_dl_display_command(result)
         replay_cmd = build_abx_dl_replay_command(result, config=config)
@@ -164,15 +176,19 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         cmd_attr = html.escape(replay_cmd, quote=True)
 
         # Build output link - use embed_path() which checks output_files first
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        embed_path = result.embed_path()
+        snapshot_id = str(result.snapshot_id)
         if embed_path and result.status == "succeeded":
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
             output_link = build_snapshot_url(snapshot_id, "", config=config)
+        output_link_attr = html.escape(output_link, quote=True)
 
         # Get version - try cmd_version field
-        version = result.cmd_version if result.cmd_version else "-"
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
 
         # Unique ID for this row's expandable output
         row_id = f"output_{idx}_{str(result.id)[:8]}"
@@ -189,31 +205,32 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 <td style="padding: 10px 12px; white-space: nowrap;">
                     <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
                                  font-size: 11px; font-weight: 600; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status}</span>
+                                 color: {color}; background: {bg};">{status_text}</span>
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{result.plugin}">
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
                     {icon}
                 </td>
-                <td style="padding: 10px 12px; font-weight: 500; color: #334155;">
-                        <a href="{output_link}" target="_blank"
+                <td class="archive-results-plugin" style="padding: 10px 12px; font-weight: 500; color: #334155;">
+                        <a href="{output_link_attr}" target="_blank"
                            style="color: #334155; text-decoration: none;"
                        title="View output fullscreen"
                        onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
                        onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
-                        {result.plugin}
+                        {plugin_text}
                     </a>
                 </td>
-                <td style="padding: 10px 12px; max-width: 280px;">
+                <td class="archive-results-output" style="padding: 10px 12px; max-width: 280px;">
                     <span onclick="document.getElementById('{row_id}').open = !document.getElementById('{row_id}').open"
-                          style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
+                          style="display: block; max-width: 280px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
+                                 color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
                           title="Click to expand full output">
                         {output_display}
                     </span>
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; color: #64748b; font-size: 12px; text-align: right;">
+                <td class="archive-results-files" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right;">
                     {output_file_count}
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; color: #64748b; font-size: 12px;">
+                <td class="archive-results-completed" style="padding: 10px 12px; color: #64748b; font-size: 12px;">
                     {end_time}
                 </td>
                 <td style="padding: 10px 12px; white-space: nowrap;">
@@ -227,7 +244,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 </td>
                 <td style="padding: 10px 8px; white-space: nowrap;">
                     <div style="display: flex; gap: 4px;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                            title="View output">📄</a>
                         <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
@@ -244,9 +261,9 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                         </summary>
                         <div style="margin-top: 8px; padding: 10px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; max-height: 200px; overflow: auto;">
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
-                                <span style="margin-right: 16px;"><b>ID:</b> <code>{str(result.id)}</code></span>
+                                <span style="margin-right: 16px;"><b>ID:</b> <code>{result.id!s}</code></span>
                                 <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
-                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{result.pwd or "-"}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
                             </div>
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <b>Output:</b>
@@ -284,17 +301,17 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         """
 
     return mark_safe(f"""
-        <div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden; background: #fff; width: 100%;">
-            <table style="width: 100%; border-collapse: collapse; font-size: 14px;">
+        <div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow-x: auto; overflow-y: hidden; background: #fff; width: 100%;">
+            <table class="archive-results-table" style="width: 100%; min-width: 1100px; border-collapse: collapse; font-size: 14px;">
                 <thead>
                     <tr style="background: #f8fafc; border-bottom: 2px solid #e2e8f0;">
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Details</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Status</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; width: 32px;"></th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
-                        <th style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
+                        <th class="archive-results-plugin" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
+                        <th class="archive-results-output" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
+                        <th class="archive-results-files" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
+                        <th class="archive-results-completed" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
@@ -314,15 +331,12 @@ class ArchiveResultInline(admin.TabularInline):
     name = "Archive Results Log"
     model = ArchiveResult
     parent_model = Snapshot
-    # fk_name = 'snapshot'
     extra = 0
     sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version")
     readonly_fields = ("id", "result_id", "completed", "command", "version")
     fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str")
-    # exclude = ('id',)
     ordering = ("end_ts",)
     show_change_link = True
-    # # classes = ['collapse']
 
     def get_parent_object_from_request(self, request):
         resolved = resolve(request.path_info)
@@ -354,11 +368,9 @@ class ArchiveResultInline(admin.TabularInline):
     def get_formset(self, request, obj=None, **kwargs):
         formset = super().get_formset(request, obj, **kwargs)
         snapshot = self.get_parent_object_from_request(request)
-        form_class = getattr(formset, "form", None)
-        base_fields = getattr(form_class, "base_fields", {})
+        base_fields = formset.form.base_fields
         snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
 
-        # import ipdb; ipdb.set_trace()
         # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
 
         # default values for new entries
@@ -404,11 +416,11 @@ class ArchiveResultAdmin(BaseModelAdmin):
     list_display_links = None
     sort_fields = ("id", "created_at", "plugin", "status")
     readonly_fields = (
-        "admin_actions",
         "cmd",
         "cmd_version",
         "pwd",
         "cmd_str",
+        "admin_actions",
         "snapshot_info",
         "tags_str",
         "created_at",
@@ -428,20 +440,13 @@ class ArchiveResultAdmin(BaseModelAdmin):
         "output_json",
         "process__cmd",
     )
-    autocomplete_fields = ["snapshot"]
+    autocomplete_fields = ("snapshot",)
 
     fieldsets = (
-        (
-            "Actions",
-            {
-                "fields": ("admin_actions",),
-                "classes": ("card", "wide"),
-            },
-        ),
         (
             "Snapshot",
             {
-                "fields": ("snapshot", "snapshot_info", "tags_str"),
+                "fields": ("snapshot", "snapshot_info", "tags_str", "admin_actions"),
                 "classes": ("card", "wide"),
             },
         ),
@@ -476,14 +481,14 @@ class ArchiveResultAdmin(BaseModelAdmin):
     )
 
     list_filter = ("status", "plugin", "start_ts")
-    ordering = ["-start_ts"]
+    ordering = ("-start_ts",)
     list_per_page = 50
 
     paginator = AcceleratedPaginator
     save_on_top = True
     show_full_result_count = False
 
-    actions = ["delete_selected"]
+    actions = ("delete_selected",)
 
     class Meta:
         verbose_name = "Archive Result"
@@ -491,12 +496,40 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         return super().change_view(request, object_id, form_url, extra_context)
 
+    def get_admin_toolbar_actions(self, request, obj):
+        if obj is None:
+            return []
+        self.request = request
+        return [
+            {
+                "label": "View Output",
+                "icon": "📄",
+                "url": self.get_output_view_url(obj),
+                "title": "Open the archived output for this result",
+            },
+            {
+                "label": "Output files",
+                "icon": "📁",
+                "url": self.get_output_files_url(obj),
+                "title": "Browse the output files for this result",
+            },
+            {
+                "label": "Download Zip",
+                "icon": "⬇",
+                "url": self.get_output_zip_url(obj),
+                "kind": "accent",
+                "css_classes": "archivebox-zip-button",
+                "onclick": "return window.archiveboxHandleZipClick(this, event);",
+                "extra_attrs": [("data-loading-label", "Preparing...")],
+                "title": "Download all output files as a zip",
+            },
+            {"label": "Snapshot", "icon": "🗂", "url": self.get_snapshot_view_url(obj), "title": "Open the parent snapshot view"},
+        ]
+
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
         self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         try:
@@ -522,7 +555,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
             super()
             .get_queryset(request)
             .defer(
-                "config",
                 "notes",
                 "output_json",
             )
@@ -573,20 +605,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return queryset.filter(reduce(and_, filters)).distinct(), True
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        output_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        config = request.archivebox_config
+        output_path = result.embed_path()
         if not output_path:
             output_path = result.plugin or ""
         return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
         return f"{self.get_output_files_url(result)}&download=zip"
@@ -606,16 +638,48 @@ class ArchiveResultAdmin(BaseModelAdmin):
             self.get_output_zip_url(result),
         )
 
+    @admin.display(description="")
+    def admin_actions(self, result):
+        return format_html(
+            """
+            <div style="display: flex; flex-wrap: wrap; gap: 12px; align-items: center;">
+                <a class="btn" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; color: #334155; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}">
+                    📄 View Output
+                </a>
+                <a class="btn" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; color: #334155; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}">
+                    📁 Output files
+                </a>
+                <a class="btn archivebox-zip-button" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #eff6ff; border: 1px solid #bfdbfe; border-radius: 8px; color: #1d4ed8; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}"
+                   data-loading-label="Preparing..."
+                   onclick="return window.archiveboxHandleZipClick(this, event);">
+                    <span class="archivebox-zip-spinner" aria-hidden="true"></span>
+                    <span class="archivebox-zip-label">⬇ Download Zip</span>
+                </a>
+                <a class="btn" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; color: #334155; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}">
+                    🗂 Snapshot
+                </a>
+            </div>
+            """,
+            self.get_output_view_url(result),
+            self.get_output_files_url(result),
+            self.get_output_zip_url(result),
+            self.get_snapshot_view_url(result),
+        )
+
     @admin.display(
         description="Snapshot",
         ordering="snapshot__url",
     )
     def snapshot_info(self, result):
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -665,7 +729,7 @@ class ArchiveResultAdmin(BaseModelAdmin):
         process = result.process_record
         if not process:
             return "-"
-        process_label = process.pid or "-"
+        process_label = get_process_link_label(process)
         return format_html(
             '<a href="{}"><code>{}</code></a>',
             reverse("admin:machine_process_change", args=[process.id]),
@@ -679,17 +743,16 @@ class ArchiveResultAdmin(BaseModelAdmin):
             return "-"
         machine = process.machine
         return format_html(
-            '<a href="{}"><code>{}</code> {}</a>',
+            '<a href="{}">{}</a>',
             reverse("admin:machine_machine_change", args=[machine.id]),
-            str(machine.id)[:8],
             machine.hostname,
         )
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        request = getattr(self, "request", None)
+        request = self.request
         display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
         return format_html(
             """
             <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -710,10 +773,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_display(self, result):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         # Determine output link path - use embed_path() which checks output_files
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+        embed_path = result.embed_path()
         output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
         snapshot_id = str(result.snapshot_id)
         return format_html(
@@ -728,12 +791,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
         if not output_text:
             return "-"
 
-        request = getattr(self, "request", None)
-        live_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        live_path = result.embed_path()
         if live_path:
             return format_html(
                 '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
                 output_text,
                 output_text,
             )
@@ -744,59 +807,19 @@ class ArchiveResultAdmin(BaseModelAdmin):
             output_text,
         )
 
-    @admin.display(description="")
-    def admin_actions(self, result):
-        return format_html(
-            """
-            <div style="display:flex; flex-wrap:wrap; gap:12px; align-items:center;">
-                <a class="btn" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#f8fafc; border:1px solid #e2e8f0; border-radius:8px; color:#334155; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   onmouseover="this.style.background='#f1f5f9'; this.style.borderColor='#cbd5e1';"
-                   onmouseout="this.style.background='#f8fafc'; this.style.borderColor='#e2e8f0';">
-                    📄 View Output
-                </a>
-                <a class="btn" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#f8fafc; border:1px solid #e2e8f0; border-radius:8px; color:#334155; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   onmouseover="this.style.background='#f1f5f9'; this.style.borderColor='#cbd5e1';"
-                   onmouseout="this.style.background='#f8fafc'; this.style.borderColor='#e2e8f0';">
-                    📁 Output files
-                </a>
-                <a class="btn archivebox-zip-button" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#eff6ff; border:1px solid #bfdbfe; border-radius:8px; color:#1d4ed8; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   data-loading-label="Preparing..."
-                   onclick="return window.archiveboxHandleZipClick(this, event);"
-                   onmouseover="this.style.background='#dbeafe'; this.style.borderColor='#93c5fd';"
-                   onmouseout="this.style.background='#eff6ff'; this.style.borderColor='#bfdbfe';">
-                    <span class="archivebox-zip-spinner" aria-hidden="true"></span>
-                    <span class="archivebox-zip-label">⬇ Download Zip</span>
-                </a>
-                <a class="btn" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#f8fafc; border:1px solid #e2e8f0; border-radius:8px; color:#334155; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   onmouseover="this.style.background='#f1f5f9'; this.style.borderColor='#cbd5e1';"
-                   onmouseout="this.style.background='#f8fafc'; this.style.borderColor='#e2e8f0';">
-                    🗂 Snapshot
-                </a>
-            </div>
-            """,
-            self.get_output_view_url(result),
-            self.get_output_files_url(result),
-            self.get_output_zip_url(result),
-            self.get_snapshot_view_url(result),
-        )
-
     def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+        snapshot_dir = Path(result.snapshot.output_dir)
         output_html = format_html(
             '<pre style="display: inline-block">{}</pre><br/>',
             result.output_str_for_display(),
         )
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         output_html += format_html(
             '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
         )
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+        embed_path = result.embed_path() or ""
         path_from_embed = snapshot_dir / (embed_path or "")
         output_html += format_html(
             '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',
@@ -808,8 +831,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
         else:
             root_dir = str(snapshot_dir)
 
-        # print(root_dir, str(list(os.walk(root_dir))))
-
         for root, dirs, files in os.walk(root_dir):
             depth = root.replace(root_dir, "").count(os.sep) + 1
             if depth > 2:

archivebox/core/admin_site.py

@@ -2,23 +2,46 @@ __package__ = "archivebox.core"
 
 from typing import TYPE_CHECKING, Any
 
-from django.contrib import admin
-from django.db import DatabaseError, connection
 from admin_data_views.admin import (
     admin_data_index_view as adv_admin_data_index_view,
+)
+from admin_data_views.admin import (
     get_admin_data_urls as adv_get_admin_data_urls,
+)
+from admin_data_views.admin import (
     get_app_list as adv_get_app_list,
 )
+from django.contrib import admin
+from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth.decorators import login_not_required
+from django.contrib.auth.views import LoginView
+from django.db import DatabaseError, connection
+from django.http import HttpResponseRedirect
+from django.urls import reverse
+from django.utils.decorators import method_decorator
+from django.utils.translation import gettext as _
+from django.views.decorators.cache import never_cache
 
 from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
+from archivebox.core.context_processors import get_static_cache_key
+from archivebox.core.routes_util import is_allowed_archivebox_redirect_url
 
 if TYPE_CHECKING:
+    from admin_data_views.typing import AppDict
     from django.http import HttpRequest
     from django.template.response import TemplateResponse
     from django.urls import URLPattern, URLResolver
 
-    from admin_data_views.typing import AppDict
+
+class ArchiveBoxLoginView(LoginView):
+    def get_redirect_url(self) -> str:
+        redirect_to = self.request.POST.get(
+            self.redirect_field_name,
+            self.request.GET.get(self.redirect_field_name),
+        )
+        if is_allowed_archivebox_redirect_url(redirect_to, request=self.request):
+            return redirect_to
+        return ""
 
 
 class ArchiveBoxAdmin(admin.AdminSite):
@@ -30,7 +53,7 @@ class ArchiveBoxAdmin(admin.AdminSite):
     def each_context(self, request: "HttpRequest") -> dict[str, Any]:
         context = super().each_context(request)
         context["VERSION"] = VERSION
-        context["STATIC_CACHE_KEY"] = (get_COMMIT_HASH() or VERSION or "dev").strip()
+        context["STATIC_CACHE_KEY"] = get_static_cache_key()
         return context
 
     @staticmethod
@@ -72,10 +95,36 @@ class ArchiveBoxAdmin(admin.AdminSite):
     def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
         return adv_admin_data_index_view(self, request, **kwargs)
 
+    @method_decorator(never_cache)
+    @login_not_required
+    def login(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
+        if request.method == "GET" and self.has_permission(request):
+            return HttpResponseRedirect(reverse("admin:index", current_app=self.name))
+
+        from django.contrib.admin.forms import AdminAuthenticationForm
+
+        context = {
+            **self.each_context(request),
+            "title": _("Log in"),
+            "subtitle": None,
+            "app_path": request.get_full_path(),
+            "username": request.user.get_username(),
+        }
+        if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
+            context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
+        context.update(extra_context or {})
+
+        index_path = reverse("admin:index", current_app=self.name)
+        request.current_app = self.name
+        return ArchiveBoxLoginView.as_view(
+            extra_context=context,
+            authentication_form=self.login_form or AdminAuthenticationForm,
+            template_name=self.login_template or "admin/login.html",
+            next_page=index_path,
+        )(request)
+
     def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
         response = super().index(request, extra_context)
-        if connection.vendor != "sqlite":
-            return response
 
         models_by_table: dict[str, list[dict[str, Any]]] = {}
         for app in response.context_data.get("app_list", []):
@@ -88,23 +137,18 @@ class ArchiveBoxAdmin(admin.AdminSite):
         if not models_by_table:
             return response
 
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute("SELECT tbl, stat FROM sqlite_stat1")
-                for table, stat in cursor.fetchall():
-                    try:
-                        count = int(str(stat).split()[0])
-                    except (IndexError, TypeError, ValueError):
-                        continue
-                    self._set_model_object_count(
-                        models_by_table,
-                        table,
-                        count,
-                        title=f"Approximate count from SQLite stats: {count:,}",
-                    )
-                    models_by_table.pop(table, None)
-        except DatabaseError:
-            pass
+        from archivebox.misc.db import approximate_row_counts
+
+        for table, count in approximate_row_counts(connection).items():
+            if table not in models_by_table:
+                continue
+            self._set_model_object_count(
+                models_by_table,
+                table,
+                count,
+                title=f"Approximate count from database stats: {count:,}",
+            )
+            models_by_table.pop(table, None)
 
         for table in list(models_by_table):
             try:
@@ -142,9 +186,9 @@ def register_admin_site():
 
     # Register admin views for each app
     # (Previously handled by ABX plugin system, now called directly)
+    from archivebox.api.admin import register_admin as register_api_admin
     from archivebox.core.admin import register_admin as register_core_admin
     from archivebox.crawls.admin import register_admin as register_crawls_admin
-    from archivebox.api.admin import register_admin as register_api_admin
     from archivebox.machine.admin import register_admin as register_machine_admin
     from archivebox.personas.admin import register_admin as register_personas_admin
     from archivebox.workers.admin import register_admin as register_workers_admin

archivebox/core/admin_snapshots.py

@@ -11,7 +11,7 @@ from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAll
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
-from django.db.models import Q, Count, Exists, OuterRef, Prefetch
+from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
@@ -30,7 +30,9 @@ from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_p
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
+from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
@@ -41,7 +43,6 @@ from archivebox.core.permissions import (
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 
 
-# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
 GLOBAL_CONTEXT = {}
 
 SNAPSHOT_PERMISSION_META = PERMISSIONS_META
@@ -70,13 +71,6 @@ class SnapshotActionForm(ActionForm):
 
         return [name.strip() for name in tags_str.split(",") if name.strip()]
 
-    # TODO: allow selecting actions for specific extractor plugins? is this useful?
-    # plugin = forms.ChoiceField(
-    #     choices=ArchiveResult.PLUGIN_CHOICES,
-    #     required=False,
-    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
-    # )
-
 
 class TagNameListFilter(admin.SimpleListFilter):
     title = "By tag name"
@@ -107,9 +101,7 @@ class SnapshotPermissionsListFilter(admin.SimpleListFilter):
     def queryset(self, request, queryset):
         value = self.value()
         if value:
-            return queryset.filter(
-                Q(permissions=value) | (Q(permissions__isnull=True) & Q(crawl__permissions=value)),
-            )
+            return queryset.filter(permissions=value)
         return queryset
 
 
@@ -190,6 +182,7 @@ class SnapshotSizeListFilter(admin.SimpleListFilter):
 class SnapshotResultHealthListFilter(admin.SimpleListFilter):
     title = "ArchiveResult status"
     parameter_name = "archiveresult_status"
+    SNAPSHOT_FIRST_VALUES = {"succeeded"}
 
     def lookups(self, request, model_admin):
         return (
@@ -199,7 +192,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
             ("failed", ">50% failed"),
             ("running", ">50% running"),
             ("pending", ">50% queued"),
-            ("backoff", ">50% waiting to retry"),
             ("noresults", ">50% noresults"),
         )
 
@@ -215,56 +207,70 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                 "succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
                 "failed": ArchiveResult.StatusChoices.FAILED,
                 "running": ArchiveResult.StatusChoices.STARTED,
-                "pending": ArchiveResult.StatusChoices.QUEUED,
+                "pending": (ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.BACKOFF),
                 "backoff": ArchiveResult.StatusChoices.BACKOFF,
                 "noresults": ArchiveResult.StatusChoices.NORESULTS,
             }
             if value in status_by_value:
-                # Start from ArchiveResult.status for every majority-status filter.
-                # The (status, snapshot_id) index keeps this plan stable regardless
-                # of which status is most common in a user's collection.
-                return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status_by_value[value]))
+                status = status_by_value[value]
+                if value in self.SNAPSHOT_FIRST_VALUES:
+                    # "succeeded" is overwhelmingly common in large collections.
+                    # Scan Snapshots in admin order and do indexed per-snapshot
+                    # probes so page 1 can stop after list_per_page matches.
+                    queryset = queryset.alias(
+                        total_results=ArchiveResult.snapshot_count_expr(),
+                        matching_results=ArchiveResult.snapshot_count_expr(status=status),
+                    ).filter(matching_results__gt=F("total_results") / 2)
+                    return queryset
+
+                # Rare statuses are faster status-first: use the
+                # (status, snapshot_id) index to find candidate snapshots.
+                snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
+                queryset = queryset.filter(pk__in=snapshot_ids)
+                queryset._archivebox_count_hint = len(snapshot_ids)
+                queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+                return queryset
         return queryset
 
 
 class SnapshotChangeList(SearchResultsChangeList):
     def __init__(self, request, *args, **kwargs):
         super().__init__(request, *args, **kwargs)
-        resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+        resolver_name = request.resolver_match.url_name
         self.embedded_changelist = request.GET.get("_embedded") == "crawl"
         self.snapshot_is_grid_view = not self.embedded_changelist and (
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
 
+    def _attach_archiveresult_summaries(self):
+        snapshot_ids = [obj.pk for obj in self.result_list]
+        if not snapshot_ids:
+            return
+
+        results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
+        rows = (
+            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
+            .order_by("snapshot_id", "plugin")
+            .values_list("snapshot_id", "plugin", "status", "output_size", "output_files")
+        )
+        for snapshot_id, plugin, status, output_size, output_files in rows.iterator(chunk_size=1000):
+            if plugin in seen_plugins[snapshot_id]:
+                continue
+            seen_plugins[snapshot_id].add(plugin)
+            results_by_snapshot[snapshot_id].append(
+                SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
+            )
+
+        for obj in self.result_list:
+            obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+
     def get_results(self, request):
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
-        else:
-            self.full_result_count = self.model_admin.get_paginator(
-                request,
-                self.model._default_manager.all().order_by(),
-                self.list_per_page,
-            ).count
-        self.show_full_result_count = True
-
-        snapshot_ids = [obj.pk for obj in self.result_list]
-        if snapshot_ids:
-            results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
-            seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
-            rows = (
-                ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
-                .order_by("snapshot_id", "plugin")
-                .values_list("snapshot_id", "plugin", "status", "output_size")
-            )
-            for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
-                if plugin in seen_plugins[snapshot_id]:
-                    continue
-                seen_plugins[snapshot_id].add(plugin)
-                results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
-
-            for obj in self.result_list:
-                obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+            self.show_full_result_count = True
+        self._attach_archiveresult_summaries()
 
 
 class SnapshotAdminForm(forms.ModelForm):
@@ -300,18 +306,13 @@ class SnapshotAdminForm(forms.ModelForm):
         instance = super().save(commit=False)
         permissions = self.cleaned_data["permissions_config"]
         config = dict(instance.config or {})
-        if permissions == instance.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
         instance.config = config
 
         # Handle tags_editor field
         if commit:
             instance.save()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
@@ -447,7 +448,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ),
     )
 
-    ordering = ["-timestamp"]
+    ordering = ["-created_at"]
     actions = [
         "add_tags",
         "remove_tags",
@@ -464,7 +465,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     paginator = AcceleratedPaginator
 
     save_on_top = True
-    show_full_result_count = False
+    show_full_result_count = True
 
     def get_changelist(self, request, **kwargs):
         return SnapshotChangeList
@@ -475,7 +476,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().get_ordering(request)
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
+        self.request = request
         extra_context = extra_context or {}
         extra_context["CONFIG"] = request.archivebox_config
         snapshot = self.get_object(request, object_id)
@@ -490,7 +491,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
         embedded_changelist = request.GET.get("_embedded") == "crawl"
         if embedded_changelist:
@@ -526,12 +526,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().lookup_allowed(lookup, value, request=request)
 
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_snapshot_files_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(obj.id), request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
 
     def get_snapshot_zip_url(self, obj: Snapshot) -> str:
         return f"{self.get_snapshot_files_url(obj)}&download=zip"
@@ -561,12 +561,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if permissions not in dict(PERMISSIONS_CHOICES):
             return HttpResponseBadRequest("Invalid permissions value")
 
-        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
         config = dict(snapshot.config or {})
-        if permissions == snapshot.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
 
         # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
         # the write lock for the permission/config change itself. safe_update()
@@ -589,18 +586,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         now = timezone.now()
         updated = 0
         batch = []
-        snapshots = (
-            queryset.select_related(None)
-            .select_related("crawl")
-            .only("id", "config", "crawl__id", "crawl__permissions")
-            .prefetch_related(None)
-        )
+        snapshots = queryset.select_related(None).only("id", "config").prefetch_related(None)
         for snapshot in snapshots.iterator(chunk_size=500):
             config = dict(snapshot.config or {})
-            if permissions == snapshot.crawl.permissions:
-                config.pop("PERMISSIONS", None)
-            else:
-                config["PERMISSIONS"] = permissions
+            config["PERMISSIONS"] = permissions
             snapshot.config = config
             snapshot.modified_at = now
             batch.append(snapshot)
@@ -621,12 +610,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             if retried:
                 messages.success(
                     request,
-                    f"Queued {retried} failed/skipped extractors for retry on this snapshot.",
+                    f"Queued {retried} failed extractors for retry on this snapshot.",
                 )
             else:
                 messages.info(
                     request,
-                    "No failed/skipped extractors were found on this snapshot.",
+                    "No failed extractors were found on this snapshot.",
                 )
 
         return redirect(snapshot.admin_change_url)
@@ -636,7 +625,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ordering_fields = self._get_ordering_fields(request)
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
-        is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
+        is_change_view = request.resolver_match.url_name == "core_snapshot_change"
         prefetches = ["tags"]
         if is_change_view:
             prefetches.append(
@@ -648,13 +637,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                         "plugin",
                         "status",
                         "output_size",
+                        "output_files",
+                    ),
+                ),
+            )
+        else:
+            prefetches.append(
+                Prefetch(
+                    "crawl",
+                    queryset=Crawl.objects.select_related("created_by").only(
+                        "id",
+                        "created_by_id",
+                        "created_by__id",
+                        "created_by__username",
                     ),
                 ),
             )
 
-        qs = super().get_queryset(request).select_related("crawl__created_by")
+        qs = super().get_queryset(request)
         if is_change_view:
-            qs = qs.defer("notes")
+            qs = qs.select_related("crawl__created_by").defer("notes")
         else:
             qs = qs.only(
                 "id",
@@ -668,13 +670,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 "fs_version",
                 "output_size",
                 "permissions",
-                "crawl__id",
-                "crawl__permissions",
-                "crawl__persona_id",
-                "crawl__status",
-                "crawl__created_by_id",
-                "crawl__created_by__id",
-                "crawl__created_by__username",
             )
         qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
@@ -688,12 +683,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="👁", ordering="permissions")
     def permissions_badge(self, obj):
-        permissions = getattr(obj, "snapshot_permissions", None)
+        permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
-            if obj.permissions:
-                permissions = obj.permissions
-            else:
-                permissions = obj.crawl.permissions
+            permissions = obj.permissions
         permissions = normalize_permissions(permissions)
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         menu_items = format_html_join(
@@ -748,31 +740,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         html = Template("""{{bookmarked_date}} (<code>{{timestamp}}</code>)""")
         return mark_safe(html.render(context))
 
-        # pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S')
-        # return f'{pretty_time} ({obj.timestamp})'
-
-    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
-    # def action(self, obj):
-    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
-    #     # action: update_snapshots
-    #     # select_across: 0
-    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
-    #     return format_html(
-    #         '''
-    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
-    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
-    #                 <input type="hidden" name="_selected_action" value="{}">
-    #                 <button name="update_snapshots">Check</button>
-    #                 <button name="update_titles">Pull title + favicon</button>
-    #                 <button name="update_snapshots">Update</button>
-    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
-    #                 <button name="delete_snapshots">Permanently delete</button>
-    #             </form>
-    #         ''',
-    #         csrf.get_token(self.request),
-    #         obj.pk,
-    #     )
-
     @admin.display(description="")
     def admin_actions(self, obj):
         summary_url = self.get_snapshot_view_url(obj)
@@ -858,8 +825,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def status_info(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
         return format_html(
             """
@@ -876,16 +843,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
-        request = getattr(self, "request", None)
-        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)
 
     @admin.display(
         description="Title",
         ordering="title",
     )
     def title_str(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         title_raw = (obj.title or "").strip()
         url_raw = (obj.url or "").strip()
         title_normalized = title_raw.lower()
@@ -938,34 +905,58 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
 
     def _get_preview_data(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         results = self._get_prefetched_results(obj)
         if results is not None:
-            has_screenshot = any(r.plugin == "screenshot" for r in results)
-            has_favicon = any(r.plugin == "favicon" for r in results)
+            results = [r for r in results if r.plugin in ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")]
         else:
-            available_plugins = set(obj.archiveresult_set.filter(plugin__in=("screenshot", "favicon")).values_list("plugin", flat=True))
-            has_screenshot = "screenshot" in available_plugins
-            has_favicon = "favicon" in available_plugins
+            results = list(
+                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_files",
+                    "output_str",
+                ),
+            )
 
-        if not has_screenshot and not has_favicon:
+        def result_output_path(result, filename: str) -> str | None:
+            output_files = result.output_files or {}
+            file_info = output_files.get(filename)
+            if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                return None
+            if file_info.get("root_relative"):
+                return filename
+            return f"{result.plugin}/{filename}"
+
+        def result_urls(plugin: str, filenames: tuple[str, ...]) -> list[str]:
+            urls: list[str] = []
+            for result in results:
+                if result.plugin != plugin or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
+                    continue
+                for filename in filenames:
+                    output_path = result_output_path(result, filename)
+                    if output_path:
+                        urls.append(build_snapshot_url(str(obj.id), output_path, request=request, config=config))
+            return urls
+
+        screenshot_urls = result_urls("screenshot", ("screenshot.png",))
+        extension_screenshot_urls = result_urls(EXTENSION_SCREENSHOT_PLUGIN, ("screenshot-1.png", "screenshot.png"))
+        favicon_urls = result_urls("favicon", ("favicon.ico",))
+
+        if not screenshot_urls and not extension_screenshot_urls and not favicon_urls:
             return None
 
-        if has_screenshot:
-            img_url = build_snapshot_url(str(obj.id), "screenshot/screenshot.png", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "screenshot.png", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+        all_screenshot_urls = [*screenshot_urls, *extension_screenshot_urls]
+        if all_screenshot_urls:
+            img_url = all_screenshot_urls[0]
+            fallbacks = [*all_screenshot_urls[1:], *favicon_urls]
             img_alt = "Screenshot"
             preview_class = "screenshot"
         else:
-            img_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+            img_url = favicon_urls[0]
+            fallbacks = favicon_urls[1:]
             img_alt = "Favicon"
             preview_class = "favicon"
 
@@ -983,6 +974,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "preview_class": preview_class,
             "onerror_js": onerror_js,
             "fallback_list": fallback_list,
+            "favicon_url": favicon_urls[0] if favicon_urls else "",
+            "favicon_fallback_list": ",".join(favicon_urls[1:]),
         }
 
     @admin.display(description="", empty_value="")
@@ -991,10 +984,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if not preview:
             return ""
 
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-        fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
+        favicon_url = preview.get("favicon_url") or ""
+        if not favicon_url:
+            return ""
+        fallback_list = preview.get("favicon_fallback_list") or ""
         onerror_js = (
             "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
             "(this.src=this.dataset.fallbacks.split(',').shift(), "
@@ -1030,8 +1023,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description=" ", empty_value="")
     def snapshot_summary(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         preview = self._get_preview_data(obj)
         stats = self._get_progress_stats(obj)
         archive_size = stats["output_size"] or 0
@@ -1095,8 +1088,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
         visible_results = sorted_results[:14]
         output = []
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         for result in visible_results:
             icon = mark_safe(get_plugin_icon(result.plugin))
             if not icon.strip():
@@ -1123,12 +1116,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             mark_safe("".join(output)),
         )
 
-    @admin.display(
-        # ordering='archiveresult_count'
-    )
+    @admin.display()
     def size(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         archive_size = self._get_progress_stats(obj)["output_size"] or 0
         if archive_size:
             size_txt = printable_filesize(archive_size)
@@ -1242,7 +1233,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def _get_progress_stats(self, obj):
-        cached_stats = getattr(obj, "_admin_progress_stats", None)
+        cached_stats = obj.__dict__.get("_admin_progress_stats")
         if cached_stats is not None:
             return cached_stats
 
@@ -1288,31 +1279,31 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def _get_prefetched_results(self, obj):
         if "_admin_archiveresults" in obj.__dict__:
             return obj.__dict__["_admin_archiveresults"]
-        if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
+        if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
             return obj.archiveresult_set.all()
         return None
 
     def _get_expected_hook_total(self, obj) -> int:
         try:
-            request = getattr(self, "request", None)
-            if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}:
+            request = self.request
+            if request.resolver_match.url_name in {"core_snapshot_changelist", "core_snapshot_change"}:
                 return 0
 
-            crawl = getattr(obj, "crawl", None)
-            snapshot_config = getattr(obj, "config", None) or {}
-            crawl_config = getattr(crawl, "config", None) or {}
+            crawl = obj.crawl
+            snapshot_config = obj.config or {}
+            crawl_config = crawl.config or {}
             has_scoped_config = bool(snapshot_config or crawl_config)
 
             if request is not None and not has_scoped_config:
-                cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
+                cached_total = request.__dict__.get("archivebox_expected_snapshot_hook_total")
                 if cached_total is None:
-                    config = getattr(request, "archivebox_config", None) or get_config()
+                    config = request.archivebox_config
                     cached_total = len(discover_hooks("Snapshot", config=config))
                     request.archivebox_expected_snapshot_hook_total = cached_total
                 return cached_total
 
             if request is not None:
-                scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+                scoped_cache = request.__dict__.get("archivebox_expected_snapshot_hook_totals_by_scope")
                 if scoped_cache is None:
                     scoped_cache = {}
                     request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
@@ -1332,8 +1323,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             return 0
 
     def _get_prefetched_tags(self, obj):
-        if hasattr(obj, "_prefetched_objects_cache") and "tags" in obj._prefetched_objects_cache:
-            return list(obj._prefetched_objects_cache["tags"])
+        prefetched_cache = obj.__dict__.get("_prefetched_objects_cache", {})
+        if "tags" in prefetched_cache:
+            return list(prefetched_cache["tags"])
         return None
 
     def _get_ordering_fields(self, request):
@@ -1374,11 +1366,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         extra_context["snapshot_is_grid_view"] = True
         return self.changelist_view(request, extra_context=extra_context)
 
-    # for debugging, uncomment this to print all requests:
-    # def changelist_view(self, request, extra_context=None):
-    #     print('[*] Got request', request.method, request.POST)
-    #     return super().changelist_view(request, extra_context=None)
-
     @admin.action(
         description="🔁 Redo Failed",
     )
@@ -1390,10 +1377,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if queued:
             messages.success(
                 request,
-                f"Queued {queued} failed/skipped extractors for retry. The background runner will process them.",
+                f"Queued {queued} failed extractors for retry. The background runner will process them.",
             )
         else:
-            messages.info(request, "No failed/skipped extractors were found in the selected snapshots.")
+            messages.info(request, "No failed extractors were found in the selected snapshots.")
 
     @admin.action(
         description="🆕 Archive Now",

archivebox/core/admin_tags.py

@@ -117,7 +117,7 @@ class TagAdmin(BaseModelAdmin):
         sort = normalize_tag_sort((request.GET.get("sort") or "created_desc").strip())
         created_by = normalize_created_by_filter((request.GET.get("created_by") or "").strip())
         year = normalize_created_year_filter((request.GET.get("year") or "").strip())
-        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "yes").strip())
+        has_snapshots = normalize_has_snapshots_filter((request.GET.get("has_snapshots") or "all").strip())
         context = {
             **self.admin_site.each_context(request),
             **(extra_context or {}),
@@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshots", ordering="num_snapshots")
     def num_snapshots(self, tag: Tag):
-        count = getattr(tag, "num_snapshots", None)
+        count = tag.__dict__.get("num_snapshots")
         if count is None:
             count = tag.snapshot_set.count()
         return format_html(

archivebox/core/admin_users.py

@@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):
 
     def snapshot_count_badge(self, obj):
         snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
-        snapshot_count = getattr(obj, "snapshot_count", 0)
+        snapshot_count = obj.__dict__.get("snapshot_count", 0)
         snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
         return format_html(
             (

archivebox/core/apps.py

@@ -20,6 +20,14 @@ class CoreConfig(AppConfig):
 
         ModelWithOutputDir.register_delete_signal()
 
+        # SQLite ignores VARCHAR(n) limits but PostgreSQL enforces them; clamp
+        # CharField values on save so writes behave the same on both backends.
+        from django.db.models.signals import pre_save
+
+        from archivebox.misc.db import truncate_overlong_charfields
+
+        pre_save.connect(truncate_overlong_charfields, dispatch_uid="archivebox_truncate_overlong_charfields")
+
         # Import models to register state machines with the registry
         # Skip during makemigrations to avoid premature state machine access
         if "makemigrations" not in sys.argv:

archivebox/core/asgi.py

@@ -7,10 +7,11 @@ For more information on this file, see
 https://docs.djangoproject.com/en/stable/howto/deployment/asgi/
 """
 
-from archivebox.config.django import setup_django
 from django.core.asgi import get_asgi_application
 
-setup_django(in_memory_db=False, check_db=True)
+from archivebox.config.django import setup_django
+
+setup_django(check_db=True)
 
 
 def _patch_thread_sensitive_context_shutdown() -> None:
@@ -73,16 +74,3 @@ _patch_thread_sensitive_context_shutdown()
 
 # Standard Django ASGI application (no websockets/channels needed)
 application = get_asgi_application()
-
-# If websocket support is needed later, install channels and use:
-# from channels.routing import ProtocolTypeRouter, URLRouter
-# from channels.auth import AuthMiddlewareStack
-# from channels.security.websocket import AllowedHostsOriginValidator
-# from archivebox.core.routing import websocket_urlpatterns
-#
-# application = ProtocolTypeRouter({
-#     "http": get_asgi_application(),
-#     "websocket": AllowedHostsOriginValidator(
-#         AuthMiddlewareStack(URLRouter(websocket_urlpatterns))
-#     ),
-# })

archivebox/core/context_processors.py

@@ -1,9 +1,21 @@
+from pathlib import Path
+
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
 
 
+def get_static_cache_key() -> str:
+    """Version the admin stylesheet even when the checkout has uncommitted edits."""
+    base_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
+    admin_css_path = Path(__file__).resolve().parent.parent / "templates" / "static" / "admin.css"
+    try:
+        return f"{base_key}-{admin_css_path.stat().st_mtime_ns}"
+    except OSError:
+        return base_key
+
+
 def archivebox_globals(request):
     return {
         "VERSION": VERSION,
-        "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(),
+        "STATIC_CACHE_KEY": get_static_cache_key(),
     }

archivebox/core/forms.py

@@ -1,17 +1,17 @@
 __package__ = "archivebox.core"
 
 import re
-from decimal import Decimal, InvalidOperation, ROUND_CEILING
+from decimal import ROUND_CEILING, Decimal, InvalidOperation
 
 from django import forms
 
-from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
-from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
-from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
-from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
+from archivebox.crawls.schedule_util import validate_schedule
+from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
+from archivebox.personas.models import Persona
 from archivebox.plugins.discovery import get_plugins
 from archivebox.plugins.forms import (
     PLUGIN_GROUP_DEFINITIONS,
@@ -19,7 +19,6 @@ from archivebox.plugins.forms import (
     PluginConfigFormMixin,
     get_choice_field,
 )
-from archivebox.personas.models import Persona
 
 DEPTH_CHOICES = (
     ("0", "depth = 0 (archive just these URLs)"),
@@ -30,20 +29,83 @@ DEPTH_CHOICES = (
 )
 
 
+def _split_strip(value: str, delimiter: str) -> list[str]:
+    return [part.strip() for part in value.split(delimiter) if part.strip()]
+
+
+def parse_tag_string(value: str | None) -> list[str]:
+    """Parse the legacy tag-editing format used by the admin tag field."""
+    if not value:
+        return []
+
+    if "," not in value and '"' not in value:
+        return sorted(set(_split_strip(value, " ")))
+
+    tags: list[str] = []
+    buffer: list[str] = []
+    deferred_chunks: list[str] = []
+    saw_unquoted_comma = False
+    in_quote = False
+    chars = iter(value)
+
+    try:
+        while True:
+            char = next(chars)
+            if char == '"':
+                if buffer:
+                    deferred_chunks.append("".join(buffer))
+                    buffer = []
+                in_quote = True
+                char = next(chars)
+                while char != '"':
+                    buffer.append(char)
+                    char = next(chars)
+                tag = "".join(buffer).strip()
+                if tag:
+                    tags.append(tag)
+                buffer = []
+                in_quote = False
+            else:
+                if not saw_unquoted_comma and char == ",":
+                    saw_unquoted_comma = True
+                buffer.append(char)
+    except StopIteration:
+        if buffer:
+            if in_quote and "," in buffer:
+                saw_unquoted_comma = True
+            deferred_chunks.append("".join(buffer))
+
+    delimiter = "," if saw_unquoted_comma else " "
+    for chunk in deferred_chunks:
+        tags.extend(_split_strip(chunk, delimiter))
+
+    return sorted(set(tags))
+
+
+def edit_string_for_tag_names(tags) -> str:
+    names = []
+    for tag in tags:
+        name = tag.name
+        names.append(f'"{name}"' if "," in name or " " in name else name)
+    return ", ".join(sorted(names))
+
+
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
+    allow_crawl_execution_config_fields = False
+
     # Basic fields
     url = forms.CharField(
         label="URLs",
-        strip=True,
+        strip=False,
         widget=forms.Textarea(
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
                 "placeholder": (
-                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
-                    "(e.g. markdown, HTML, etc.). Examples:\n"
-                    "https://example.com\n"
-                    "https://news.ycombinator.com,https://news.google.com\n"
-                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                    "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, RSS, embedded in text, etc.\n\n"
+                    "Examples:\n\n"
+                    "https://example.com\n\n"
+                    "https://news.ycombinator.com,https://news.google.com\n\n"
+                    "Or any text-based format [containing URLs](https://github.com/ArchiveBox/ArchiveBox)..."
                 ),
             },
         ),
@@ -236,7 +298,10 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def __init__(self, *args, **kwargs):
         self.request = kwargs.pop("request", None)
-        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
+        self.can_override_crawl_config = False
+        if self.request is not None:
+            user = self.request.user
+            self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
         default_persona = Persona.get_or_create_default()
@@ -312,12 +377,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def clean_url(self):
         value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
             raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value
 
     def clean_url_filters(self):
         from archivebox.crawls.models import Crawl
@@ -423,7 +485,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 class TagWidget(forms.TextInput):
     def format_value(self, value):
         if value is not None and not isinstance(value, str):
-            value = edit_string_for_tags(value)
+            value = edit_string_for_tag_names(value)
         return super().format_value(value)
 
 
@@ -432,12 +494,7 @@ class TagField(forms.CharField):
 
     def clean(self, value):
         value = super().clean(value)
-        try:
-            return parse_tags(value)
-        except ValueError:
-            raise forms.ValidationError(
-                "Please provide a comma-separated list of tags.",
-            )
+        return parse_tag_string(value)
 
     def has_changed(self, initial, data):
         # Always return False if the field is disabled since self.bound_data

archivebox/core/middleware.py

@@ -3,35 +3,35 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
-from django.utils import timezone
+
+from django.conf import settings
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
-from django.core.exceptions import ImproperlyConfigured
-from django.shortcuts import redirect
 from django.contrib.staticfiles import finders
-from django.utils.http import http_date
+from django.core.exceptions import ImproperlyConfigured
 from django.http import HttpResponseForbidden, HttpResponseNotModified
+from django.shortcuts import redirect
+from django.utils import timezone
+from django.utils.http import http_date
 
-from archivebox.config.common import get_config
 from archivebox.config import VERSION
+from archivebox.config.common import get_config
 from archivebox.config.version import get_COMMIT_HASH
 from archivebox.core.routes_util import (
-    build_snapshot_url,
     build_admin_url,
+    build_snapshot_url,
     build_web_url,
-    get_api_host,
     get_admin_host,
+    get_api_host,
     get_base_host,
     get_listen_host,
     get_listen_subdomain,
-    get_public_host,
     get_web_host,
     host_matches,
     is_snapshot_subdomain,
     split_host_port,
 )
-from archivebox.core.views import SnapshotHostView, OriginalDomainHostView
-
+from archivebox.core.views import OriginalDomainHostView, SnapshotHostView
 
 ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 
@@ -42,7 +42,7 @@ def _admin_login_hint_cookie_domain(config) -> str | None:
     NOTE: this cookie carries only the single bit "user is logged in on
     admin somewhere"; it MUST NOT be confused with the session cookie,
     which stays admin-host-scoped (see core/settings.py
-    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
 
     Returns the hostname portion of ``get_base_host`` (which respects
     ``BASE_URL`` and falls back to the local-bind mapping). Strips the
@@ -73,7 +73,6 @@ def detect_timezone(request, activate: bool = True):
         tz = timezone.get_fixed_timezone(int(gmt_offset))
         if activate:
             timezone.activate(tz)
-    # print('GMT_OFFSET', gmt_offset, tz)
     return tz
 
 
@@ -85,6 +84,39 @@ def TimezoneMiddleware(get_response):
     return middleware
 
 
+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None or config.SERVER_SECURITY_MODE == "auto":
+            from archivebox.config.common import get_request_config
+
+            config = get_request_config(request, resolve_plugins=False)
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        if not config.BASE_URL and request.path.startswith("/admin/"):
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        request_hostname, _request_port = split_host_port(request_host)
+        if host_matches(request_hostname, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_hostname, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
     snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -117,15 +149,15 @@ def CacheControlMiddleware(get_response):
                     response.headers["Last-Modified"] = http_date(mtime)
                 return response
 
-        if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
-            if not response.get("Cache-Control"):
-                config = getattr(request, "archivebox_config", None)
-                if config is None:
-                    config = get_config(resolve_plugins=False)
-                    request.archivebox_config = config
-                policy = "private" if config.PERMISSIONS == "private" else "public"
-                response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
-                # print('Set Cache-Control header to', response['Cache-Control'])
+        if ("/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path)) and not response.get(
+            "Cache-Control",
+        ):
+            config = request.__dict__.get("archivebox_config")
+            if config is None:
+                config = get_config(resolve_plugins=False)
+                request.archivebox_config = config
+            policy = "private" if config.PERMISSIONS == "private" else "public"
+            response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
         return response
 
     return middleware
@@ -136,10 +168,29 @@ def ServerSecurityModeMiddleware(get_response):
     allowed_methods = {"GET", "HEAD", "OPTIONS"}
 
     def middleware(request):
-        config = getattr(request, "archivebox_config", None)
-        if config is None:
-            config = get_config(resolve_plugins=False)
-            request.archivebox_config = config
+        config = request.__dict__.get("archivebox_config")
+        if config is None or config.SERVER_SECURITY_MODE == "auto":
+            base_config = config or get_config(resolve_plugins=False)
+            if base_config.SERVER_SECURITY_MODE == "auto" and request.method.upper() not in allowed_methods:
+                request_host, _request_port = split_host_port((request.get_host() or "").lower())
+                base_host, _base_port = split_host_port(get_base_host(config=base_config))
+                admin_host, _admin_port = split_host_port(get_admin_host(config=base_config))
+                api_host, _api_port = split_host_port(get_api_host(config=base_config))
+                web_host, _web_port = split_host_port(get_web_host(config=base_config))
+                control_hosts = {host for host in (base_host, admin_host, api_host, web_host) if host}
+                first_run_setup_request = not base_config.BASE_URL and (
+                    request.path == "/admin/login/"
+                    or (
+                        bool(getattr(request.user, "is_superuser", False))
+                        and re.fullmatch(r"/admin/machine/machine/[0-9a-f-]{32,36}/change/", request.path)
+                    )
+                )
+                if control_hosts and request_host not in control_hosts and not first_run_setup_request:
+                    return HttpResponseForbidden("ArchiveBox is running with the control plane disabled on this host.")
+
+            from archivebox.config.common import get_request_config
+
+            config = get_request_config(request, resolve_plugins=False)
 
         if config.CONTROL_PLANE_ENABLED:
             return get_response(request)
@@ -163,20 +214,23 @@ def HostRoutingMiddleware(get_response):
     snapshot_path_re = re.compile(
         r"^/(?P<username>[^/]+)/(?P<date>\d{4}(?:\d{2})?(?:\d{2})?)/(?P<domain>[^/]+)/(?P<snapshot_id>[0-9a-fA-F-]{8,36})(?:/(?P<path>.*))?$",
     )
+    snapshot_replay_path_re = re.compile(
+        r"^/snapshot/(?P<snapshot_id>[0-9a-fA-F-]{8,36})(?:/(?P<path>.*))?$",
+    )
+    original_replay_path_re = re.compile(r"^/original/(?P<domain>[^/]+)(?:/(?P<path>.*))?$")
 
     def middleware(request):
         if request.path in {"/health", "/health/"}:
             return get_response(request)
 
         request_host = (request.get_host() or "").lower()
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
         admin_host = get_admin_host(config=config)
         web_host = get_web_host(config=config)
         api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
         listen_host = get_listen_host(config=config)
         subdomain = get_listen_subdomain(request_host, config=config)
 
@@ -185,25 +239,7 @@ def HostRoutingMiddleware(get_response):
         if request.path.startswith("/static/") or request.path in {"/favicon.ico", "/robots.txt"}:
             return get_response(request)
 
-        # In subdomain mode with no explicit BASE_URL we can't safely emit
-        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
-        # request's own Host (via the request-host fallback in get_base_url),
-        # so prepending ``admin.`` to whatever the client sent produces a
-        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
-        # through; the misconfig banner on the rendered page tells the user
-        # to pin BASE_URL so the redirects can resume.
-        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
-            return get_response(request)
-
-        if config.USES_SUBDOMAIN_ROUTING and not host_matches(request_host, admin_host):
-            # ``/add`` is admin-only unless ``PUBLIC_ADD_VIEW`` is on. Without
-            # this redirect, hitting it on public.* falls into AddView's
-            # auth check, bounces through ``/accounts/login/?next=/add/`` →
-            # ``/admin/login/?next=/add/``, and Django admin's LoginView
-            # silently drops ``next`` when the user already has an admin
-            # session — dumping the user on the admin homepage instead of
-            # the add form. Routing the request to admin.* directly lets
-            # AddView run on the host where the session lives.
+        if config.USES_SUBDOMAIN_ROUTING and config.BASE_URL and not host_matches(request_host, admin_host):
             add_should_redirect = not config.PUBLIC_ADD_VIEW and (request.path == "/add" or request.path.startswith("/add/"))
             if (
                 request.path == "/admin"
@@ -217,18 +253,52 @@ def HostRoutingMiddleware(get_response):
                     target = f"{target}?{request.META['QUERY_STRING']}"
                 return redirect(target)
 
+        if subdomain and is_snapshot_subdomain(subdomain):
+            view = SnapshotHostView.as_view()
+            return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
+
+        # In subdomain mode with no explicit BASE_URL we can't safely emit
+        # ``admin.``/``web.``/``snap-*.`` redirects: every URL builder uses the
+        # request's own Host (via the request-host fallback in get_base_url),
+        # so prepending ``admin.`` to whatever the client sent produces a
+        # redirect chain of ``admin.admin.admin.<host>``. Pass the request
+        # through; the misconfig banner on the rendered page tells the user
+        # to pin BASE_URL so the redirects can resume.
+        if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
+            return get_response(request)
+
+        if config.USES_SUBDOMAIN_ROUTING:
+            snapshot_replay_match = snapshot_replay_path_re.match(request.path)
+            if snapshot_replay_match:
+                target = build_snapshot_url(
+                    snapshot_replay_match.group("snapshot_id"),
+                    (snapshot_replay_match.group("path") or "").strip("/"),
+                    request=request,
+                )
+                if request.META.get("QUERY_STRING"):
+                    target = f"{target}?{request.META['QUERY_STRING']}"
+                return redirect(target)
+
+            original_replay_match = original_replay_path_re.match(request.path)
+            if original_replay_match:
+                view = OriginalDomainHostView.as_view()
+                return view(
+                    request,
+                    domain=original_replay_match.group("domain"),
+                    path=(original_replay_match.group("path") or "").strip("/"),
+                )
+
         if not config.USES_SUBDOMAIN_ROUTING:
             if host_matches(request_host, listen_host):
                 return get_response(request)
 
             req_host, req_port = split_host_port(request_host)
             listen_host_only, listen_port = split_host_port(listen_host)
-            if req_host.endswith(f".{listen_host_only}"):
-                if not listen_port or not req_port or listen_port == req_port:
-                    target = build_web_url(request.path, request=request)
-                    if request.META.get("QUERY_STRING"):
-                        target = f"{target}?{request.META['QUERY_STRING']}"
-                    return redirect(target)
+            if req_host.endswith(f".{listen_host_only}") and (not listen_port or not req_port or listen_port == req_port):
+                target = build_web_url(request.path, request=request)
+                if request.META.get("QUERY_STRING"):
+                    target = f"{target}?{request.META['QUERY_STRING']}"
+                return redirect(target)
 
             return get_response(request)
 
@@ -282,16 +352,7 @@ def HostRoutingMiddleware(get_response):
             request._cached_user = request.user
             return get_response(request)
 
-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
         if subdomain:
-            if is_snapshot_subdomain(subdomain):
-                view = SnapshotHostView.as_view()
-                return view(request, snapshot_id=subdomain, path=request.path.lstrip("/"))
             view = OriginalDomainHostView.as_view()
             return view(request, domain=subdomain, path=request.path.lstrip("/"))
 
@@ -323,7 +384,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
     header = "HTTP_REMOTE_USER"
 
     def process_request(self, request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config

archivebox/core/migrations/0004_auto_20200713_1552.py

@@ -9,6 +9,12 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        # The original schema allowed NULL timestamps. Use the already-unique
+        # 32-character snapshot ID before enforcing the later NOT NULL field.
+        migrations.RunSQL(
+            sql="UPDATE core_snapshot SET timestamp = id WHERE timestamp IS NULL",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
         migrations.AlterField(
             model_name="snapshot",
             name="timestamp",

archivebox/core/migrations/0007_archiveresult.py

@@ -1,9 +1,11 @@
 # Generated by Django 3.0.8 on 2020-11-04 12:25
 
 import json
+from datetime import timezone
 from pathlib import Path
 
 from django.db import migrations, models
+from django.utils.dateparse import parse_datetime
 import django.db.models.deletion
 
 # Handle old vs new import paths
@@ -30,6 +32,13 @@ except AttributeError:
     JSONField = jsonfield.JSONField
 
 
+def parse_legacy_datetime(value):
+    parsed = parse_datetime(value) if isinstance(value, str) else value
+    if parsed is not None and parsed.tzinfo is None:
+        return parsed.replace(tzinfo=timezone.utc)
+    return parsed
+
+
 def forwards_func(apps, schema_editor):
     Snapshot = apps.get_model("core", "Snapshot")
     ArchiveResult = apps.get_model("core", "ArchiveResult")
@@ -55,8 +64,8 @@ def forwards_func(apps, schema_editor):
                         pwd=result["pwd"],
                         cmd=result.get("cmd") or [],
                         cmd_version=result.get("cmd_version") or "unknown",
-                        start_ts=result["start_ts"],
-                        end_ts=result["end_ts"],
+                        start_ts=parse_legacy_datetime(result["start_ts"]),
+                        end_ts=parse_legacy_datetime(result["end_ts"]),
                         status=result["status"],
                         output=result.get("output") or "null",
                     )

archivebox/core/migrations/0023_upgrade_to_0_9_0.py

@@ -19,6 +19,19 @@ def get_table_columns(table_name):
     return {row[1] for row in cursor.fetchall()}
 
 
+def assert_rebuild_row_count(cursor, source_table, destination_table, expected_count=None):
+    """Refuse to drop a legacy table unless every source row was copied."""
+    if expected_count is None:
+        cursor.execute(f"SELECT COUNT(*) FROM {source_table}")
+        expected_count = cursor.fetchone()[0]
+    cursor.execute(f"SELECT COUNT(*) FROM {destination_table}")
+    destination_count = cursor.fetchone()[0]
+    if destination_count != expected_count:
+        raise RuntimeError(
+            f"Refusing to replace {source_table}: copied {destination_count} of {expected_count} rows",
+        )
+
+
 def normalize_cmd(cmd):
     if not cmd:
         return "[]"
@@ -47,6 +60,14 @@ def normalize_status(status):
 
 def upgrade_core_tables(apps, schema_editor):
     """Upgrade core tables from v0.7.2 or v0.8.6rc0 to v0.9.0."""
+    # sqlite-only legacy repair/rebuild. On postgres this raw SQL (PRAGMA,
+    # sqlite_master, INSERT OR IGNORE, DATETIME, table-rebuild dance) is both
+    # invalid and unnecessary: a postgres database can never contain legacy
+    # data at this point. The final _pg_sync_schema op resyncs the real
+    # postgres schema to migration state instead.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
     from archivebox.uuid_compat import uuid7
 
     cursor = connection.cursor()
@@ -91,6 +112,7 @@ def upgrade_core_tables(apps, schema_editor):
             output VARCHAR(1024),
             created_at DATETIME,
             modified_at DATETIME,
+            notes TEXT NOT NULL DEFAULT '',
 
             FOREIGN KEY (snapshot_id) REFERENCES core_snapshot(id) ON DELETE CASCADE
         );
@@ -115,6 +137,8 @@ def upgrade_core_tables(apps, schema_editor):
             archiveresult_select_cols.append("created_at")
         if has_archiveresult_modified_at:
             archiveresult_select_cols.append("modified_at")
+        if "notes" in archiveresult_cols:
+            archiveresult_select_cols.append("notes")
 
         if has_uuid and not has_abid:
             # Migrating from v0.7.2+ (has uuid column)
@@ -124,6 +148,8 @@ def upgrade_core_tables(apps, schema_editor):
                 select_cols.append("created_at")
             if has_archiveresult_modified_at:
                 select_cols.append("modified_at")
+            if "notes" in archiveresult_cols:
+                select_cols.append("notes")
             cursor.execute(f"SELECT {', '.join(select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
             for i, record in enumerate(old_records, start=1):
@@ -139,8 +165,8 @@ def upgrade_core_tables(apps, schema_editor):
                     INSERT OR IGNORE INTO core_archiveresult_new (
                         id, uuid, snapshot_id, cmd, pwd, cmd_version,
                         start_ts, end_ts, status, extractor, output,
-                        created_at, modified_at
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        created_at, modified_at, notes
+                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 """,
                     (
                         values["id"],
@@ -156,6 +182,7 @@ def upgrade_core_tables(apps, schema_editor):
                         values["output"] or "",
                         values.get("created_at") or start_ts,
                         values.get("modified_at") or end_ts,
+                        values.get("notes") or "",
                     ),
                 )
                 if i % PROGRESS_EVERY == 0:
@@ -178,8 +205,8 @@ def upgrade_core_tables(apps, schema_editor):
                     INSERT OR IGNORE INTO core_archiveresult_new (
                         uuid, snapshot_id, cmd, pwd, cmd_version,
                         start_ts, end_ts, status, extractor, output,
-                        created_at, modified_at
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        created_at, modified_at, notes
+                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 """,
                     (
                         new_uuid,
@@ -194,6 +221,7 @@ def upgrade_core_tables(apps, schema_editor):
                         values["output"] or "",
                         values.get("created_at") or start_ts,
                         values.get("modified_at") or end_ts,
+                        values.get("notes") or "",
                     ),
                 )
                 if i % PROGRESS_EVERY == 0:
@@ -213,8 +241,8 @@ def upgrade_core_tables(apps, schema_editor):
                     INSERT OR IGNORE INTO core_archiveresult_new (
                         id, uuid, snapshot_id, cmd, pwd, cmd_version,
                         start_ts, end_ts, status, extractor, output,
-                        created_at, modified_at
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        created_at, modified_at, notes
+                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 """,
                     (
                         values["id"],
@@ -230,12 +258,14 @@ def upgrade_core_tables(apps, schema_editor):
                         values["output"] or "",
                         values.get("created_at") or start_ts,
                         values.get("modified_at") or end_ts,
+                        values.get("notes") or "",
                     ),
                 )
                 if i % PROGRESS_EVERY == 0:
                     print(f"        copied {i}/{len(old_records)} ArchiveResults...")
         print(f"      copied {len(old_records)} ArchiveResults")
 
+    assert_rebuild_row_count(cursor, "core_archiveresult", "core_archiveresult_new", row_count)
     cursor.execute("DROP TABLE IF EXISTS core_archiveresult;")
     cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult;")
 
@@ -288,13 +318,29 @@ def upgrade_core_tables(apps, schema_editor):
             snapshot_cols = get_table_columns("core_snapshot")
             has_added = "added" in snapshot_cols
             has_bookmarked_at = "bookmarked_at" in snapshot_cols
+            inferred_status_sql = """
+                CASE
+                    WHEN EXISTS (
+                        SELECT 1 FROM core_archiveresult
+                        WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                        AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                    )
+                    THEN 'queued'
+                    WHEN EXISTS (
+                        SELECT 1 FROM core_archiveresult
+                        WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                    )
+                    THEN 'sealed'
+                    ELSE 'queued'
+                END
+            """
 
             if has_added and not has_bookmarked_at:
                 # Migrating from v0.7.2 (has added/updated fields)
                 print("      copying Snapshots from 0.7.x schema...")
                 # timestamp is the legacy bookmark/import timestamp and archive/{timestamp} identity.
                 # added is the DB row creation/import time, and updated was renamed to downloaded_at in 0.8.x.
-                cursor.execute("""
+                cursor.execute(f"""
                     INSERT OR IGNORE INTO core_snapshot_new (
                         id, url, timestamp, title,
                         bookmarked_at, created_at, modified_at, downloaded_at,
@@ -313,20 +359,7 @@ def upgrade_core_tables(apps, schema_editor):
                         COALESCE(added, CURRENT_TIMESTAMP) as created_at,
                         COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
                         updated as downloaded_at,
-                        CASE
-                            WHEN EXISTS (
-                                SELECT 1 FROM core_archiveresult
-                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
-                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
-                            )
-                            THEN 'queued'
-                            WHEN EXISTS (
-                                SELECT 1 FROM core_archiveresult
-                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
-                            )
-                            THEN 'sealed'
-                            ELSE 'queued'
-                        END as status
+                        {inferred_status_sql} as status
                     FROM core_snapshot;
                 """)
                 print(f"      copied {cursor.rowcount} Snapshots")
@@ -344,6 +377,9 @@ def upgrade_core_tables(apps, schema_editor):
                 if has_crawl_id:
                     insert_cols.append("crawl_id")
                     select_cols.append("REPLACE(crawl_id, '-', '')")
+                if "parent_snapshot_id" in snapshot_cols:
+                    insert_cols.append("parent_snapshot_id")
+                    select_cols.append("REPLACE(parent_snapshot_id, '-', '')")
                 if has_status:
                     insert_cols.append("status")
                     select_cols.append(
@@ -365,9 +401,24 @@ def upgrade_core_tables(apps, schema_editor):
                         END
                         """,
                     )
+                else:
+                    insert_cols.append("status")
+                    select_cols.append(inferred_status_sql)
                 if has_retry_at:
                     insert_cols.append("retry_at")
                     select_cols.append("retry_at")
+                for field_name in (
+                    "depth",
+                    "fs_version",
+                    "config",
+                    "notes",
+                    "num_uses_succeeded",
+                    "num_uses_failed",
+                    "current_step",
+                ):
+                    if field_name in snapshot_cols:
+                        insert_cols.append(field_name)
+                        select_cols.append(field_name)
 
                 cursor.execute(f"""
                     INSERT OR IGNORE INTO core_snapshot_new ({", ".join(insert_cols)})
@@ -378,6 +429,7 @@ def upgrade_core_tables(apps, schema_editor):
             else:
                 print(f"Warning: Unexpected Snapshot schema - has_added={has_added}, has_bookmarked_at={has_bookmarked_at}")
 
+    assert_rebuild_row_count(cursor, "core_snapshot", "core_snapshot_new")
     cursor.execute("DROP TABLE IF EXISTS core_snapshot;")
     cursor.execute("ALTER TABLE core_snapshot_new RENAME TO core_snapshot;")
 
@@ -416,6 +468,7 @@ def upgrade_core_tables(apps, schema_editor):
         tag_has_data = cursor.fetchone()[0] > 0
 
         if tag_has_data:
+            tag_cols = get_table_columns("core_tag")
             cursor.execute("PRAGMA table_info(core_tag)")
             tag_id_type = None
             for row in cursor.fetchall():
@@ -427,22 +480,23 @@ def upgrade_core_tables(apps, schema_editor):
                 # v0.8.6rc0: Tag IDs are UUIDs, need to convert to INTEGER
                 print("      converting Tag IDs from UUID to integers...")
 
-                # Get all tags with their UUIDs
-                cursor.execute("SELECT id, name, slug, created_at, modified_at, created_by_id FROM core_tag ORDER BY name")
+                # Get all tags with their UUIDs and any available audit fields.
+                tag_select_cols = ["id", "name", "slug"]
+                tag_select_cols.extend(field for field in ("created_at", "modified_at", "created_by_id") if field in tag_cols)
+                cursor.execute(f"SELECT {', '.join(tag_select_cols)} FROM core_tag ORDER BY name")
                 tags = cursor.fetchall()
 
                 # Create mapping from old UUID to new INTEGER ID
                 uuid_to_int_map = {}
                 for i, tag in enumerate(tags, start=1):
-                    old_id, name, slug, created_at, modified_at, created_by_id = tag
+                    values = dict(zip(tag_select_cols, tag))
+                    old_id = values.pop("id")
                     uuid_to_int_map[old_id] = i
                     # Insert with new INTEGER ID
+                    insert_values = {"id": i, **values}
                     cursor.execute(
-                        """
-                        INSERT OR IGNORE INTO core_tag_new (id, name, slug, created_at, modified_at, created_by_id)
-                        VALUES (?, ?, ?, ?, ?, ?)
-                    """,
-                        (i, name, slug, created_at, modified_at, created_by_id),
+                        f"INSERT OR IGNORE INTO core_tag_new ({', '.join(insert_values)}) VALUES ({', '.join('%s' for _ in insert_values)})",
+                        tuple(insert_values.values()),
                     )
                     if i % PROGRESS_EVERY == 0:
                         print(f"        copied {i}/{len(tags)} Tags...")
@@ -463,23 +517,33 @@ def upgrade_core_tables(apps, schema_editor):
                             cursor.execute(
                                 """
                                 INSERT OR IGNORE INTO core_snapshot_tags (id, snapshot_id, tag_id)
-                                VALUES (?, ?, ?)
+                                VALUES (%s, %s, %s)
                             """,
                                 (st_id, snapshot_id, new_tag_id),
                             )
                         if i % PROGRESS_EVERY == 0:
                             print(f"        copied {i}/{len(snapshot_tags)} SnapshotTag rows...")
+                    cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
+                    copied_snapshot_tags = cursor.fetchone()[0]
+                    if copied_snapshot_tags != len(snapshot_tags):
+                        raise RuntimeError(
+                            f"Refusing to replace Tag IDs: copied {copied_snapshot_tags} of {len(snapshot_tags)} SnapshotTag rows",
+                        )
                 print(f"      copied {len(tags)} Tags")
             else:
-                # v0.7.2: Tag IDs are already INTEGER
-                print("      copying Tags from 0.7.x schema...")
-                cursor.execute("""
-                    INSERT OR IGNORE INTO core_tag_new (id, name, slug)
-                    SELECT id, name, slug
+                # v0.7.x and early v0.8.x use INTEGER IDs. Preserve audit
+                # metadata when the source schema has it.
+                print("      copying Tags with integer IDs...")
+                copy_cols = ["id", "name", "slug"]
+                copy_cols.extend(field for field in ("created_at", "modified_at", "created_by_id") if field in tag_cols)
+                cursor.execute(f"""
+                    INSERT OR IGNORE INTO core_tag_new ({", ".join(copy_cols)})
+                    SELECT {", ".join(copy_cols)}
                     FROM core_tag;
                 """)
                 print(f"      copied {cursor.rowcount} Tags")
 
+    assert_rebuild_row_count(cursor, "core_tag", "core_tag_new")
     cursor.execute("DROP TABLE IF EXISTS core_tag;")
     cursor.execute("ALTER TABLE core_tag_new RENAME TO core_tag;")
 
@@ -491,6 +555,15 @@ def upgrade_core_tables(apps, schema_editor):
         print("    ✓ Core table rebuild complete")
 
 
+def _pg_sync_schema(apps, schema_editor):
+    # On postgres, upgrade_core_tables is a no-op above, so the raw sqlite
+    # rebuilds of core_tag/core_snapshot/core_archiveresult never ran. Resync
+    # the real schema to this migration's end-state (tables are empty on pg).
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["Tag", "Snapshot", "SnapshotTag", "ArchiveResult"])
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("core", "0022_auto_20231023_2008"),
@@ -571,4 +644,5 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0024_assign_default_crawl.py

@@ -4,94 +4,9 @@
 from django.db import migrations, models
 
 
-def create_default_crawl_and_assign_snapshots(apps, schema_editor):
-    """
-    Create a default crawl for migrated snapshots and assign all snapshots without a crawl to it.
-    Uses raw SQL because the app registry isn't fully populated during migrations.
-    """
-    from django.db import connection
-    import uuid as uuid_lib
-    from datetime import datetime
-
-    cursor = connection.cursor()
-
-    # Check if there are any snapshots without a crawl
-    cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id IS NULL")
-    snapshots_without_crawl = cursor.fetchone()[0]
-
-    if snapshots_without_crawl == 0:
-        print("✓ Fresh install or all snapshots already have crawls")
-        return
-
-    cursor.execute("SELECT url FROM core_snapshot WHERE crawl_id IS NULL ORDER BY bookmarked_at, timestamp")
-    crawl_urls = "\n".join(url for (url,) in cursor.fetchall() if url)
-
-    # Get or create system user (pk=1)
-    cursor.execute("SELECT id FROM auth_user WHERE id = 1")
-    if not cursor.fetchone():
-        cursor.execute(
-            """
-            INSERT INTO auth_user (id, password, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)
-            VALUES (1, '!', 1, 'system', '', '', '', 1, 1, ?)
-        """,
-            [datetime.now().isoformat()],
-        )
-
-    # Create a default crawl for migrated snapshots.
-    # Depending on migration graph order, later crawls migrations may already
-    # have removed output_dir by the time this data migration runs.
-    crawl_id = uuid_lib.uuid4().hex
-    now = datetime.now().isoformat()
-    cursor.execute("PRAGMA table_info(crawls_crawl)")
-    crawl_columns = {row[1] for row in cursor.fetchall()}
-    default_columns: list[str] = []
-    default_values: list[str] = []
-    if "output_dir" in crawl_columns:
-        default_columns.append("output_dir")
-        default_values.append("''")
-    for column in ("max_size", "max_urls", "crawl_max_size", "snapshot_max_size"):
-        if column in crawl_columns:
-            default_columns.append(column)
-            default_values.append("0")
-    default_columns_sql = f"{', '.join(default_columns)}, " if default_columns else ""
-    default_values_sql = f"{', '.join(default_values)}, " if default_values else ""
-
-    cursor.execute(
-        f"""
-        INSERT INTO crawls_crawl (
-            id, created_at, modified_at, num_uses_succeeded, num_uses_failed,
-            urls, max_depth, tags_str, label, notes, {default_columns_sql}
-            status, retry_at, created_by_id, schedule_id, config, persona_id
-        ) VALUES (?, ?, ?, 0, 0, ?, 0, '', 'Migrated from v0.7.2/v0.8.6',
-                  'Auto-created crawl for migrated snapshots', {default_values_sql}
-                  'sealed', ?, 1, NULL, '{{}}', NULL)
-    """,
-        [crawl_id, now, now, crawl_urls, now],
-    )
-
-    # Assign all snapshots without a crawl to the default crawl
-    cursor.execute("UPDATE core_snapshot SET crawl_id = ? WHERE crawl_id IS NULL", [crawl_id])
-
-    print(f"✓ Assigned {snapshots_without_crawl} snapshots to default crawl {crawl_id}")
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("core", "0023_upgrade_to_0_9_0"),
-        ("crawls", "0002_upgrade_from_0_8_6"),
-        ("auth", "0012_alter_user_first_name_max_length"),
-    ]
-
-    operations = [
-        migrations.RunPython(
-            create_default_crawl_and_assign_snapshots,
-            reverse_code=migrations.RunPython.noop,
-        ),
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                # Now make crawl_id NOT NULL
-                migrations.RunSQL(
-                    sql="""
+# Raw sqlite table-rebuild that makes core_snapshot.crawl_id NOT NULL. Kept
+# byte-for-byte and replayed through Django's own RunSQL only on sqlite.
+_MAKE_CRAWL_ID_NOT_NULL_SQL = """
                         -- Rebuild snapshot table with NOT NULL crawl_id
                         CREATE TABLE core_snapshot_final (
                             id TEXT PRIMARY KEY NOT NULL,
@@ -153,8 +68,122 @@ class Migration(migrations.Migration):
                         CREATE INDEX core_snapshot_retry_at_idx ON core_snapshot(retry_at);
                         CREATE INDEX core_snapshot_created_at_idx ON core_snapshot(created_at);
                         CREATE UNIQUE INDEX core_snapshot_url_crawl_unique ON core_snapshot(url, crawl_id);
-                    """,
-                    reverse_sql=migrations.RunSQL.noop,
+                    """
+
+
+def _make_crawl_id_not_null(apps, schema_editor):
+    # sqlite-only table rebuild. Reuse Django's own RunSQL statement splitting so
+    # the sqlite behavior is byte-for-byte identical to the original RunSQL op.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+    migrations.RunSQL(sql=_MAKE_CRAWL_ID_NOT_NULL_SQL).database_forwards("core", schema_editor, None, None)
+
+
+def _pg_sync_schema(apps, schema_editor):
+    # On postgres the raw sqlite rebuilds above are skipped and crawl was only
+    # added to migration state; resync the real schema to this migration's
+    # end-state (empty tables). Snapshot is referenced by SnapshotTag and
+    # ArchiveResult, so those are rebuilt too to restore FK constraints.
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["Snapshot", "SnapshotTag", "ArchiveResult"])
+
+
+def create_default_crawl_and_assign_snapshots(apps, schema_editor):
+    """
+    Create a default crawl for migrated snapshots and assign all snapshots without a crawl to it.
+    Uses raw SQL because the app registry isn't fully populated during migrations.
+    """
+    # Legacy-data-only and sqlite-specific because it uses PRAGMA introspection.
+    # postgres install core_snapshot has no crawl_id column yet and there are no
+    # unassigned snapshots, so gate before touching any SQL.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
+    from django.db import connection
+    import uuid as uuid_lib
+    from datetime import datetime
+
+    cursor = connection.cursor()
+
+    # Check if there are any snapshots without a crawl
+    cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id IS NULL")
+    snapshots_without_crawl = cursor.fetchone()[0]
+
+    if snapshots_without_crawl == 0:
+        print("✓ Fresh install or all snapshots already have crawls")
+        return
+
+    cursor.execute("SELECT url FROM core_snapshot WHERE crawl_id IS NULL ORDER BY bookmarked_at, timestamp")
+    crawl_urls = "\n".join(url for (url,) in cursor.fetchall() if url)
+
+    # Get or create system user (pk=1)
+    cursor.execute("SELECT id FROM auth_user WHERE id = 1")
+    if not cursor.fetchone():
+        cursor.execute(
+            """
+            INSERT INTO auth_user (id, password, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)
+            VALUES (1, '!', 1, 'system', '', '', '', 1, 1, %s)
+        """,
+            [datetime.now().isoformat()],
+        )
+
+    # Create a default crawl for migrated snapshots.
+    # Depending on migration graph order, later crawls migrations may already
+    # have removed output_dir by the time this data migration runs.
+    crawl_id = uuid_lib.uuid4().hex
+    now = datetime.now().isoformat()
+    cursor.execute("PRAGMA table_info(crawls_crawl)")
+    crawl_columns = {row[1] for row in cursor.fetchall()}
+    default_columns: list[str] = []
+    default_values: list[str] = []
+    if "output_dir" in crawl_columns:
+        default_columns.append("output_dir")
+        default_values.append("''")
+    for column in ("max_size", "max_urls", "crawl_max_size", "snapshot_max_size"):
+        if column in crawl_columns:
+            default_columns.append(column)
+            default_values.append("0")
+    default_columns_sql = f"{', '.join(default_columns)}, " if default_columns else ""
+    default_values_sql = f"{', '.join(default_values)}, " if default_values else ""
+
+    cursor.execute(
+        f"""
+        INSERT INTO crawls_crawl (
+            id, created_at, modified_at, num_uses_succeeded, num_uses_failed,
+            urls, max_depth, tags_str, label, notes, {default_columns_sql}
+            status, retry_at, created_by_id, schedule_id, config, persona_id
+        ) VALUES (%s, %s, %s, 0, 0, %s, 0, '', 'Migrated from v0.7.2/v0.8.6',
+                  'Auto-created crawl for migrated snapshots', {default_values_sql}
+                  'sealed', %s, 1, NULL, '{{}}', NULL)
+    """,
+        [crawl_id, now, now, crawl_urls, now],
+    )
+
+    # Assign all snapshots without a crawl to the default crawl
+    cursor.execute("UPDATE core_snapshot SET crawl_id = %s WHERE crawl_id IS NULL", [crawl_id])
+
+    print(f"✓ Assigned {snapshots_without_crawl} snapshots to default crawl {crawl_id}")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0023_upgrade_to_0_9_0"),
+        ("crawls", "0002_upgrade_from_0_8_6"),
+        ("auth", "0012_alter_user_first_name_max_length"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            create_default_crawl_and_assign_snapshots,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                # Now make crawl_id NOT NULL (sqlite-only table rebuild)
+                migrations.RunPython(
+                    _make_crawl_id_not_null,
+                    reverse_code=migrations.RunPython.noop,
                 ),
             ],
             state_operations=[
@@ -169,4 +198,5 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py

@@ -11,6 +11,12 @@ from archivebox.uuid_compat import uuid7
 
 def copy_old_fields_to_new(apps, schema_editor):
     """Copy data from old field names to new field names after AddField operations."""
+    # sqlite-only legacy data copy (PRAGMA introspection). On postgres there is
+    # no legacy data and the state-only AddFields below are resynced by
+    # _pg_sync_schema at the end of this migration.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
     cursor = connection.cursor()
 
     # Check if old fields still exist
@@ -43,6 +49,20 @@ def copy_old_fields_to_new(apps, schema_editor):
     # NOTE: UUIDs are already populated by migration 0023 for all migration paths.
 
 
+def _pg_sync_schema(apps, schema_editor):
+    # This migration mixes real ORM AddFields with state-only AddFields (the
+    # snapshot config/current_step/depth/notes/num_uses_*/parent_snapshot/
+    # retry_at/status columns and archiveresult created_at/modified_at were
+    # created on sqlite by 0023's raw rebuild, so here they are state-only).
+    # On postgres those columns would never land, desyncing the real schema.
+    # Resync to this migration's final state (empty tables on pg). Tag and
+    # Snapshot both gain state-only fields; Snapshot is referenced by
+    # SnapshotTag and ArchiveResult, so rebuild those too.
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["Tag", "Snapshot", "SnapshotTag", "ArchiveResult"])
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("core", "0024_assign_default_crawl"),
@@ -61,8 +81,8 @@ class Migration(migrations.Migration):
         ),
         # NOTE: RemoveField for cmd, cmd_version, pwd moved to migration 0027
         # to allow data migration to Process records first
-        # NOTE: created_at/modified_at are created by migration 0023 so they can
-        # preserve old ArchiveResult row metadata before SQLite table rebuilds.
+        # NOTE: created_at/modified_at/notes are created by migration 0023 so
+        # they can preserve old ArchiveResult metadata before SQLite rebuilds.
         # Update Django's state here before the AddField operations below.
         migrations.SeparateDatabaseAndState(
             database_operations=[],
@@ -77,6 +97,11 @@ class Migration(migrations.Migration):
                     name="modified_at",
                     field=models.DateTimeField(auto_now=True),
                 ),
+                migrations.AddField(
+                    model_name="archiveresult",
+                    name="notes",
+                    field=models.TextField(blank=True, default=""),
+                ),
             ],
         ),
         migrations.AddField(
@@ -95,11 +120,6 @@ class Migration(migrations.Migration):
                 max_length=255,
             ),
         ),
-        migrations.AddField(
-            model_name="archiveresult",
-            name="notes",
-            field=models.TextField(blank=True, default=""),
-        ),
         migrations.AddField(
             model_name="archiveresult",
             name="num_uses_failed",
@@ -218,26 +238,33 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
-        migrations.AddField(
-            model_name="tag",
-            name="created_at",
-            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, null=True),
-        ),
-        migrations.AddField(
-            model_name="tag",
-            name="created_by",
-            field=models.ForeignKey(
-                default=archivebox.base_models.models.get_or_create_system_user_pk,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                related_name="tag_set",
-                to=settings.AUTH_USER_MODEL,
-            ),
-        ),
-        migrations.AddField(
-            model_name="tag",
-            name="modified_at",
-            field=models.DateTimeField(auto_now=True),
+        # 0023 creates these columns before rebuilding the legacy Tag table so
+        # their values survive. Declare them in state without rebuilding it.
+        migrations.SeparateDatabaseAndState(
+            database_operations=[],
+            state_operations=[
+                migrations.AddField(
+                    model_name="tag",
+                    name="created_at",
+                    field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, null=True),
+                ),
+                migrations.AddField(
+                    model_name="tag",
+                    name="created_by",
+                    field=models.ForeignKey(
+                        default=archivebox.base_models.models.get_or_create_system_user_pk,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="tag_set",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="tag",
+                    name="modified_at",
+                    field=models.DateTimeField(auto_now=True),
+                ),
+            ],
         ),
         # Copy data from old field names to new field names after AddField operations
         migrations.RunPython(
@@ -335,4 +362,5 @@ class Migration(migrations.Migration):
             model_name="snapshot",
             constraint=models.UniqueConstraint(fields=("timestamp",), name="unique_timestamp"),
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0027_copy_archiveresult_to_process.py

@@ -1,7 +1,7 @@
 # Generated by hand on 2026-01-01
 # Copies ArchiveResult cmd/pwd/cmd_version data to Process records before removing old fields
 
-from django.db import migrations, connection
+from django.db import migrations
 import json
 from pathlib import Path
 from archivebox.uuid_compat import uuid7
@@ -51,7 +51,7 @@ def get_or_create_current_machine(cursor):
     guid = f"host_{hostname}"  # Simple but stable identifier
 
     # Check if machine exists
-    cursor.execute("SELECT id FROM machine_machine WHERE guid = ?", [guid])
+    cursor.execute("SELECT id FROM machine_machine WHERE guid = %s", [guid])
     row = cursor.fetchone()
 
     if row:
@@ -76,7 +76,7 @@ def get_or_create_current_machine(cursor):
                 hw_in_docker, hw_in_vm, hw_manufacturer, hw_product, hw_uuid,
                 os_arch, os_family, os_platform, os_release, os_kernel,
                 stats, config, num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 0, 0, '', '', '',
+            ) VALUES (%s, %s, %s, %s, %s, 0, 0, '', '', '',
                       '', '', '', '', '', '{}', '{}', 0, 0)
         """,
             [machine_id, now, now, guid, hostname],
@@ -90,7 +90,7 @@ def get_or_create_current_machine(cursor):
                 hw_in_docker, hw_in_vm, hw_manufacturer, hw_product, hw_uuid,
                 os_arch, os_family, os_platform, os_release, os_kernel,
                 stats, num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 0, 0, '', '', '',
+            ) VALUES (%s, %s, %s, %s, %s, 0, 0, '', '', '',
                       '', '', '', '', '', '{}', 0, 0)
         """,
             [machine_id, now, now, guid, hostname],
@@ -125,7 +125,7 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
     cursor.execute(
         """
         SELECT id FROM machine_binary
-        WHERE machine_id = ? AND name = ? AND abspath = ? AND version = ?
+        WHERE machine_id = %s AND name = %s AND abspath = %s AND version = %s
     """,
         [machine_id, name, abspath, version],
     )
@@ -162,7 +162,7 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
         "num_uses_succeeded": 0,
     }
     insert_cols = [col for col in values_by_col if col in binary_cols]
-    placeholders = ", ".join(["?"] * len(insert_cols))
+    placeholders = ", ".join(["%s"] * len(insert_cols))
     cursor.execute(
         f"""
         INSERT INTO machine_binary ({", ".join(insert_cols)})
@@ -244,7 +244,7 @@ def create_process(cursor, machine_id, pwd, cmd, status, exit_code, started_at,
         "num_uses_succeeded": 0,
     }
     insert_cols = [col for col in values_by_col if col in process_cols]
-    placeholders = ", ".join(["?"] * len(insert_cols))
+    placeholders = ", ".join(["%s"] * len(insert_cols))
     cursor.execute(
         f"""
         INSERT INTO machine_process ({", ".join(insert_cols)})
@@ -275,7 +275,15 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     - failed → exited (exit_code=1)
     - skipped → exited (exit_code=None)
     """
-    cursor = connection.cursor()
+    # sqlite-only legacy data copy because it uses PRAGMA introspection.
+    # A postgres install can never contain legacy ArchiveResult cmd/pwd data at
+    # this point, so there is nothing to copy into machine_process. The
+    # RemoveField ops below are real (non-state-only) and drop the now-empty
+    # cmd/pwd/cmd_version columns on both vendors, so no resync is needed.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
+    cursor = schema_editor.connection.cursor()
 
     # Check if old fields still exist (skip if fresh install or already migrated)
     cursor.execute("PRAGMA table_info(core_archiveresult)")
@@ -287,96 +295,101 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
 
     # Check if process_id field exists (should exist from 0026)
     if "process_id" not in cols:
-        print("✗ ERROR: process_id field not found. Migration 0026 must run first.")
-        return
+        raise RuntimeError("process_id field not found. Migration 0026 must run first.")
 
     # Get or create Machine.current()
     machine_id = get_or_create_current_machine(cursor)
 
-    # Get ArchiveResults without process_id that have cmd data
+    # Get every ArchiveResult without a Process. Rows with empty execution
+    # metadata still need a Process before the legacy columns can be removed.
     # Use plugin (extractor was renamed to plugin in migration 0025)
     cursor.execute("""
         SELECT id, snapshot_id, plugin, cmd, pwd, cmd_version,
                status, start_ts, end_ts, created_at
         FROM core_archiveresult
         WHERE process_id IS NULL
-        AND (cmd IS NOT NULL OR pwd IS NOT NULL)
     """)
 
     results = cursor.fetchall()
 
     if not results:
         print("    ✓ No ArchiveResults need Process migration")
-        return
-
-    print(f"    - Migrating {len(results)} ArchiveResults to Process rows...")
+    else:
+        print(f"    - Migrating {len(results)} ArchiveResults to Process rows...")
 
     migrated_count = 0
-    skipped_count = 0
-    error_count = 0
-
     for i, row in enumerate(results):
         ar_id, snapshot_id, plugin, cmd_raw, pwd, cmd_version, status, start_ts, end_ts, created_at = row
 
-        try:
-            # Parse cmd field
-            cmd_array = parse_cmd_field(cmd_raw)
+        # Parse cmd field
+        cmd_array = parse_cmd_field(cmd_raw)
 
-            # Extract binary info from cmd[0] if available
-            binary_id = None
-            if cmd_array and cmd_array[0]:
-                binary_name = Path(cmd_array[0]).name or plugin  # Fallback to plugin name
-                binary_abspath = cmd_array[0]
-                binary_version = cmd_version or ""
+        # Extract binary info from cmd[0] if available
+        binary_id = None
+        if cmd_array and cmd_array[0]:
+            binary_name = Path(cmd_array[0]).name or plugin  # Fallback to plugin name
+            binary_abspath = cmd_array[0]
+            binary_version = cmd_version or ""
 
-                # Get or create Binary record
-                binary_id = get_or_create_binary(
-                    cursor,
-                    machine_id,
-                    binary_name,
-                    binary_abspath,
-                    binary_version,
-                )
-
-            # Map status
-            process_status, exit_code = map_status(status)
-
-            # Set timestamps
-            started_at = start_ts or created_at
-            ended_at = end_ts if process_status == "exited" else None
-
-            # Create Process record
-            process_id = create_process(
-                cursor=cursor,
-                machine_id=machine_id,
-                pwd=pwd or "",
-                cmd=cmd_array,
-                status=process_status,
-                exit_code=exit_code,
-                started_at=started_at,
-                ended_at=ended_at,
-                binary_id=binary_id,
+            # Get or create Binary record
+            binary_id = get_or_create_binary(
+                cursor,
+                machine_id,
+                binary_name,
+                binary_abspath,
+                binary_version,
             )
+        elif cmd_version:
+            raise RuntimeError(f"ArchiveResult {ar_id} has cmd_version metadata but no command")
 
-            # Link ArchiveResult to Process
-            cursor.execute(
-                "UPDATE core_archiveresult SET process_id = ? WHERE id = ?",
-                [process_id, ar_id],
-            )
+        # Map status
+        process_status, exit_code = map_status(status)
 
-            migrated_count += 1
-            if migrated_count % PROGRESS_EVERY == 0:
-                print(f"      migrated {migrated_count}/{len(results)} ArchiveResults...")
+        # Set timestamps
+        started_at = start_ts or created_at
+        ended_at = end_ts if process_status == "exited" else None
 
-        except Exception as e:
-            print(f"✗ Error migrating ArchiveResult {ar_id}: {e}")
-            import traceback
+        # Create Process record
+        process_id = create_process(
+            cursor=cursor,
+            machine_id=machine_id,
+            pwd=pwd or "",
+            cmd=cmd_array,
+            status=process_status,
+            exit_code=exit_code,
+            started_at=started_at,
+            ended_at=ended_at,
+            binary_id=binary_id,
+        )
 
-            traceback.print_exc()
-            error_count += 1
-            continue
+        # Link ArchiveResult to Process
+        cursor.execute(
+            "UPDATE core_archiveresult SET process_id = %s WHERE id = %s",
+            [process_id, ar_id],
+        )
+        if cursor.rowcount != 1:
+            raise RuntimeError(f"ArchiveResult {ar_id} was not linked to Process {process_id}")
 
-    print(f"    ✓ Process migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")
+        migrated_count += 1
+        if migrated_count % PROGRESS_EVERY == 0:
+            print(f"      migrated {migrated_count}/{len(results)} ArchiveResults...")
+
+    cursor.execute("""
+        SELECT ar.id, ar.cmd, ar.pwd, ar.cmd_version, ar.process_id,
+               process.cmd, process.pwd, binary.version
+        FROM core_archiveresult AS ar
+        LEFT JOIN machine_process AS process ON process.id = ar.process_id
+        LEFT JOIN machine_binary AS binary ON binary.id = process.binary_id
+    """)
+    for ar_id, cmd_raw, pwd, cmd_version, process_id, process_cmd_raw, process_pwd, binary_version in cursor.fetchall():
+        if not process_id:
+            raise RuntimeError(f"ArchiveResult {ar_id} is not linked to a Process")
+        if parse_cmd_field(process_cmd_raw) != parse_cmd_field(cmd_raw) or process_pwd != (pwd or ""):
+            raise RuntimeError(f"ArchiveResult {ar_id} execution metadata was not preserved")
+        if cmd_version and binary_version != cmd_version:
+            raise RuntimeError(f"ArchiveResult {ar_id} command version was not preserved")
+
+    print(f"    ✓ Process migration complete: {migrated_count} migrated")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py

@@ -1,7 +1,7 @@
 # Generated by hand on 2026-01-02
 # Migrate ArchiveResult from integer PK to UUID PK (matching Snapshot)
 
-from django.db import migrations, models, connection
+from django.db import migrations, models
 from uuid import UUID
 from archivebox.uuid_compat import uuid7
 
@@ -26,7 +26,13 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
 
     Result: Clean schema with ONLY id as UUIDField (no old_id, no uuid)
     """
-    cursor = connection.cursor()
+    # sqlite-only table rebuild (sqlite_master, PRAGMA, table copy/rename). On
+    # postgres this is unnecessary and invalid: the table is empty and the
+    # state flip of id -> UUIDField is applied by _pg_sync_schema below.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
+    cursor = schema_editor.connection.cursor()
 
     # Check if table exists and has data
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'")
@@ -95,23 +101,35 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     col_names = [col[1] for col in columns]
     has_uuid_column = "uuid" in col_names
 
+    used_uuids = set()
+
+    def fresh_uuid():
+        while True:
+            candidate = uuid7().hex
+            if candidate not in used_uuids:
+                used_uuids.add(candidate)
+                return candidate
+
+    def unique_uuid(existing_uuid):
+        try:
+            candidate = UUID(str(existing_uuid)).hex if existing_uuid else None
+        except (TypeError, ValueError, AttributeError):
+            candidate = None
+
+        if candidate and candidate not in used_uuids:
+            used_uuids.add(candidate)
+            return candidate
+        return fresh_uuid()
+
     if has_uuid_column:
         cursor.execute("SELECT id, uuid FROM core_archiveresult")
         records = cursor.fetchall()
-        id_to_uuid = {}
-        for old_id, existing_uuid in records:
-            if existing_uuid:
-                # Normalize existing UUID to 32-char hex format (Django SQLite UUIDField format)
-                # (existing UUIDs might be stored with or without dashes in old schema)
-                id_to_uuid[old_id] = UUID(existing_uuid).hex
-            else:
-                # Generate new UUIDv7 (time-ordered) as 32-char hex
-                id_to_uuid[old_id] = uuid7().hex
+        id_to_uuid = {old_id: unique_uuid(existing_uuid) for old_id, existing_uuid in records}
     else:
         # 0.7.x path: no uuid column, generate new UUIDs for all records
         cursor.execute("SELECT id FROM core_archiveresult")
         records = cursor.fetchall()
-        id_to_uuid = {old_id: uuid7().hex for (old_id,) in records}
+        id_to_uuid = {old_id: fresh_uuid() for (old_id,) in records}
 
     # Step 3: Copy data with UUIDs as new primary key
     cursor.execute("SELECT * FROM core_archiveresult")
@@ -119,7 +137,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
 
     # col_names already fetched in Step 2
     inserted_count = 0
-    for i, record in enumerate(old_records):
+    for record in old_records:
         old_id = record[col_names.index("id")]
         new_uuid = id_to_uuid[old_id]
 
@@ -152,29 +170,34 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
         # Build INSERT statement (only copy fields that exist in source)
         existing_fields = [f for f in fields_to_copy if f in values]
 
-        placeholders = ", ".join(["?"] * (len(existing_fields) + 1))  # +1 for id
+        placeholders = ", ".join(["%s"] * (len(existing_fields) + 1))  # +1 for id
         field_list = "id, " + ", ".join(existing_fields)
 
         insert_values = [new_uuid] + [values.get(f) for f in existing_fields]
 
-        try:
-            cursor.execute(
-                f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})",
-                insert_values,
-            )
-            inserted_count += 1
-            if inserted_count % PROGRESS_EVERY == 0:
-                print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
-        except Exception as e:
-            print(f"[0029] ERROR inserting record {old_id}: {e}")
-            if i == 0:
-                print(f"[0029] First record values: {insert_values[:5]}...")
-                raise
+        cursor.execute(
+            f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})",
+            insert_values,
+        )
+        inserted_count += 1
+        if inserted_count % PROGRESS_EVERY == 0:
+            print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
 
     if old_records:
         print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults")
 
-    # Step 4: Replace old table with new table
+    cursor.execute("SELECT COUNT(*) FROM core_archiveresult")
+    source_count = cursor.fetchone()[0]
+    cursor.execute("SELECT COUNT(*) FROM core_archiveresult_new")
+    destination_count = cursor.fetchone()[0]
+    if source_count != row_count or destination_count != row_count or inserted_count != row_count:
+        raise RuntimeError(
+            "ArchiveResult UUID migration row-count mismatch: "
+            f"expected={row_count}, source={source_count}, destination={destination_count}, inserted={inserted_count}",
+        )
+
+    # Step 4: Replace old table with new table. Django runs this migration in
+    # one transaction, so any failure above leaves the source table untouched.
     cursor.execute("DROP TABLE core_archiveresult")
     cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult")
 
@@ -190,6 +213,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     print(f"    ✓ ArchiveResult UUID primary key migration complete ({row_count} records)")
 
 
+def _pg_sync_schema(apps, schema_editor):
+    # On postgres the sqlite table rebuild above is skipped; the id -> UUIDField
+    # flip and uuid-field removal only reach migration state. Resync the real
+    # (empty) core_archiveresult table to this migration's end-state. Nothing in
+    # core references ArchiveResult, so no other models need rebuilding.
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["ArchiveResult"])
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("core", "0028_alter_snapshot_fs_version"),
@@ -217,4 +250,5 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0045_archiveresult_unique_hook.py

@@ -2,13 +2,13 @@ from django.db import migrations, models
 
 
 def deduplicate_archiveresults_per_hook(apps, schema_editor):
-    """Drop duplicate ArchiveResult rows per (snapshot, plugin, hook_name).
+    """Give historical duplicate rows unique hook identities.
 
     Real long-lived collections (cabbage's demo, beta-tester DBs) accumulated
-    multiple rows per hook over the dev rc chain. The next operation adds a
-    UniqueConstraint on that tuple; without this cleanup pass the constraint
-    fails with ``UNIQUE constraint failed`` mid-migration and bricks startup.
-    Keep the row with the highest id (most recent) for each tuple.
+    multiple rows per hook. Preserve every row while keeping the newest row on
+    the canonical hook identity expected by the current scheduler. Older rows
+    receive deterministic historical identities before the existing unique
+    constraint is added.
     """
     ArchiveResult = apps.get_model("core", "ArchiveResult")
     duplicate_groups = (
@@ -20,9 +20,10 @@ def deduplicate_archiveresults_per_hook(apps, schema_editor):
             "plugin": group["plugin"],
             "hook_name": group["hook_name"],
         }
-        keep = ArchiveResult.objects.filter(**lookup).order_by("-id").values_list("id", flat=True).first()
-        if keep is not None:
-            ArchiveResult.objects.filter(**lookup).exclude(id=keep).delete()
+        rows = list(ArchiveResult.objects.filter(**lookup).order_by("-id").values_list("id", flat=True))
+        for row_id in rows[1:]:
+            historical_name = f"__legacy_history__{row_id}"[:255]
+            ArchiveResult.objects.filter(id=row_id).update(hook_name=historical_name)
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0046_repair_snapshot_permissions.py

@@ -13,6 +13,13 @@ def _repair_snapshot_permissions(apps, schema_editor):
     no underlying column. Fresh installs added the column via 0041 and
     this is a no-op.
     """
+    # sqlite-only legacy repair (PRAGMA table_xinfo + sqlite VIRTUAL generated
+    # column syntax). On postgres the permissions column was created by 0041's
+    # portable GeneratedField AddField (STORED generated column), so there is
+    # nothing to repair.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
     cursor = schema_editor.connection.cursor()
     # ``table_xinfo`` lists STORED/VIRTUAL generated columns; ``table_info``
     # silently drops them, so a prior 0041 that landed the STORED column

archivebox/core/migrations/0048_remove_archiveresult_config.py

@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0047_archiveresult_status_snapshot_index"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="archiveresult",
+            name="config",
+        ),
+    ]

archivebox/core/migrations/0049_alter_snapshot_url.py

@@ -0,0 +1,17 @@
+# Generated by Django 6.0.6 on 2026-06-04 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0048_remove_archiveresult_config"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="url",
+            field=models.TextField(db_index=True),
+        ),
+    ]