Bump for abx-dl 1.12.126

This reverts commit 848a4bea10.

.claude/settings.json

@@ -1,5 +0,0 @@
-{
-  "enabledPlugins": {
-    "pyright-lsp@claude-plugins-official": true
-  }
-}

.claude/settings.local.json

@@ -1,51 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(**)",
-      "Glob(**)",
-      "Grep(**)",
-      "Bash(python -m archivebox:*)",
-      "Bash(ls:*)",
-      "Bash(xargs:*)",
-      "Bash(python -c:*)",
-      "Bash(printf:*)",
-      "Bash(pkill:*)",
-      "Bash(python3:*)",
-      "Bash(sqlite3:*)",
-      "WebFetch(domain:github.com)",
-      "Bash(uv add:*)",
-      "Bash(mkdir:*)",
-      "Bash(chmod:*)",
-      "Bash(python -m forum_dl:*)",
-      "Bash(archivebox manage migrate:*)",
-      "Bash(cat:*)",
-      "Bash(python archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py:*)",
-      "Bash(forum-dl:*)",
-      "Bash(pip uninstall:*)",
-      "Bash(python:*)",
-      "Bash(source .venv/bin/activate)",
-      "Bash(mv:*)",
-      "Bash(echo:*)",
-      "Bash(grep:*)",
-      "WebFetch(domain:python-statemachine.readthedocs.io)",
-      "Bash(./bin/run_plugin_tests.sh:*)",
-      "Bash(done)",
-      "Bash(coverage erase:*)",
-      "Bash(gh api:*)"
-    ]
-  },
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Bash",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); if [ -n \"$REPO_ROOT\" ] && [ \"$PWD\" != \"$REPO_ROOT\" ]; then echo \"ERROR: Not in repo root ($REPO_ROOT). Current dir: $PWD\" >&2; exit 1; fi",
-            "statusMessage": "Checking working directory..."
-          }
-        ]
-      }
-    ]
-  }
-}

.dockerignore

@@ -13,10 +13,14 @@ __pycache__
 .pytest_cache/
 .ruff_cache/
 .uv-cache/
+.cache/
+.cache*/
 .github/
 .pdm-build/
+.pdm-build/**
 .pdm-python
 .eggs/
+*.egg-info/
 .git/
 !.git/
 .git/*
@@ -42,6 +46,7 @@ chromeprofile/
 chrome_profile/
 lib/
 out/
+tests/out/
 users/
 archive/
 crawls/
@@ -49,6 +54,8 @@ snapshots/
 logs/
 archivebox-docker-smoke*/
 archivebox-compose-smoke*/
+docker-test/
+docker-test*/
 core
 *.core
 
@@ -61,6 +68,8 @@ dist/
 brew_dist/
 deb_dist/
 pip_dist/
+cache/
+caches/
 assets/
 docker/
 website/

.github/ISSUE_TEMPLATE/1-bug_report.yml

@@ -69,7 +69,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/ISSUE_TEMPLATE/2-feature_request.yml

@@ -71,7 +71,7 @@ body:
         0.8.6
         ArchiveBox v0.8.6rc0 COMMIT_HASH=721427a BUILD_TIME=2024-10-21 12:57:02 1729515422
         IN_DOCKER=False IN_QEMU=False ARCH=arm64 OS=Darwin PLATFORM=macOS-15.1-arm64-arm-64bit PYTHON=Cpython (venv)
-        EUID=502:20 UID=502:20 PUID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
+        EUID=502:20 UID=502:20 FS_UID=502:20 FS_PERMS=644 FS_ATOMIC=True FS_REMOTE=False
         DEBUG=False IS_TTY=True SUDO=False ID=dfa11485:aa78ad45 SEARCH_BACKEND=ripgrep LDAP=False
         
         Binary Dependencies:

.github/SECURITY.md

@@ -2,18 +2,15 @@
 
 ---
 
-## Security Information
+## Security Documentation
 
-Please see this wiki page for important notices about ArchiveBox security, publishing your archives securely, and the dangers of executing archived JS:
-
-https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
-
-Also see this section of the README about important caveats when running ArchiveBox:
-
-https://github.com/ArchiveBox/ArchiveBox?tab=readme-ov-file#caveats
-
-You can also read these pages for more information about ArchiveBox's internals, development environment, DB schema, and more:
+Please start by reading these links to understand our existing security model, known issues, API surfaces, and setup steps:
 
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview
+- https://github.com/ArchiveBox/ArchiveBox/security/advisories/
+- https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication
 - https://github.com/ArchiveBox/ArchiveBox#archive-layout
 - https://github.com/ArchiveBox/ArchiveBox#archivebox-development
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
@@ -23,12 +20,49 @@ You can also read these pages for more information about ArchiveBox's internals,
 
 ## Reporting a Vulnerability
 
-We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.
+~We use Github's built-in [Private Reporting](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability) feature to accept vulnerability reports.~ We no longer offer CVEs or accept private reporting, 95% of the reports we received were low quality AI slop, so now we just ask for normal Github Issues or PRs.
 
-1. Go to the Security tab on our Github repo: https://github.com/ArchiveBox/ArchiveBox/security
+You can contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp) if you have a truly urgent security report, but we do not issue public CVEs.
 
-2. Click the ["Report a Vulnerability"](https://github.com/ArchiveBox/ArchiveBox/security/advisories/new) button
+---
 
-3. Fill out the form to submit the details of the report and it will be securely sent to the maintainers
+## CVEs and Report Credits
 
-You can also contact the maintainers via our public [Zulip Chat Server zulip.archivebox.io](https://zulip.archivebox.io) or [Twitter DMs @ArchiveBoxApp](https://twitter.com/ArchiveBoxApp).
+> [!IMPORTANT]
+> **Reports that only affect `dev` or beta versions will be converted to normal Github issues and will not be issued CVEs.**  
+> Reporters **will still receive credit in release notes** however.
+
+> [!TIP]
+> ~Reports that affect **stable published versions will be issued CVEs** and published w/ full credit.~
+
+Please read our existing published [security advisories](https://github.com/ArchiveBox/ArchiveBox/security/advisories), [Security Overview Docs](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview), and [issues](https://github.com/ArchiveBox/ArchiveBox/issues) and avoid creating duplicates.
+
+---
+
+## Threat Model
+
+### All admin users are assumed to be trusted
+
+ArchiveBox does not support granular user permissions, there are only two types of users available in the Admin UI & REST API:
+
+- logged in admin superusers (which are assumed to have `root` on the ArchiveBox host machine already)
+- non-logged in anonymous guest users
+
+Because admin / REST API / CLI / Python API users are already assumed to have `root` on the host machine, there are deliberately no protections against admins inputting malicious config values, attempts to attack other admin users, or destructive/malicious actions. Permissions for non-logged in users are managed via the [`PERMISSIONS`](https://github.com/archiveBox/archiveBox/wiki/configuration#permissions) + [`PUBLIC_INDEX` + `PUBLIC_ADD_VIEW`](https://github.com/archiveBox/archiveBox/wiki/configuration#public_index--public_add_view) + other config options.
+
+### Sanitization of Config Values, Filesystem Paths, DB Fields, etc.
+
+Non-logged in users can only choose between public Persona config presets on the `/add/` page, but they should never be allowed to directly set config options like `*_ARGS`, `*_BINARY` or DB fields like `url`, `title`, `tag.name`, `notes` without sanitization/filtering on save and/or use.
+
+Non-logged in users should be tightly limited to prevent attacks like SQL/shell injection, path traversal attacks, XSS, cookie leaks between admin/non-admin domains, etc. Any reports related to non-logged-in users will be prioritized as this is a surface we care a lot about hardening.
+
+### XSS, CSRF, CORS, CSP
+
+`< v0.9.x` versions do not include any XSS/CSRF protections at all, as documented in the [readme](https://github.com/ArchiveBox/ArchiveBox/tree/v0.7.4#security-risks-of-viewing-archived-js), [wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#security-concerns), [issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239), and [existing advisory](https://github.com/ArchiveBox/ArchiveBox/security/advisories/GHSA-cr45-98w9-gwqx). Do not open advisories related to XSS or CSRF for those versions, as they are missing fundamental architectural changes added in `v0.9.0` needed to support per-snapshot replay isolation.
+
+[`SERVER_SECURITY_MODE`](https://github.com/archiveBox/archiveBox/wiki/configuration#server_security_mode) + CSRF/XSS defenses were only added in the new 0.9.0 `dev` work and are planned to land in `main` in mid-2026.
+
+### SSRF / URL Validation
+
+Note archivebox is **designed to be able to archive any URL the server can reach**, including internal intranet URLs, private IPs, and localhost. Users are expected to configure `URL_ALLOWLIST`/`URL_DENYLIST` or proper firewall/ACL-level blocking at the network level if they need to limit what the server should see.
+It does a basic check to try and avoid recursively archiving snapshots hosted by the archivebox server itself, but it's not a strict security check, it's just to prevent wasted CPU/disk from recursively archiving snapshots in a browser that has the ArchiveBox Browser Extension turned on.

.github/configs/ci-linux-build.json

@@ -0,0 +1,81 @@
+{
+  "properties": {
+    "CC_BINARY": {
+      "default": "cc"
+    },
+    "LDAPSEARCH_BINARY": {
+      "default": "ldapsearch"
+    },
+    "INITDB_BINARY": {
+      "default": "initdb"
+    },
+    "PG_CTL_BINARY": {
+      "default": "pg_ctl"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{CC_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "build-essential"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{LDAPSEARCH_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "env": {
+          "version": [
+            "ldapsearch",
+            "-VV"
+          ]
+        },
+        "apt": {
+          "install_args": [
+            "ldap-utils",
+            "python3-dev",
+            "python3-setuptools",
+            "libssl-dev",
+            "libldap2-dev",
+            "libsasl2-dev",
+            "zlib1g-dev",
+            "libatomic1"
+          ],
+          "version": [
+            "ldapsearch",
+            "-VV"
+          ]
+        }
+      }
+    }
+  ],
+  "postgres_binaries": [
+    {
+      "name": "{INITDB_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "postgresql"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{PG_CTL_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "postgresql"
+          ]
+        }
+      }
+    }
+  ]
+}

.github/configs/ci-macos-brew.json

@@ -0,0 +1,13 @@
+{
+  "properties": {
+    "BREW_BINARY": {
+      "default": "brew"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{BREW_BINARY}",
+      "binproviders": "env"
+    }
+  ]
+}

.github/configs/ci-macos-build.json

@@ -0,0 +1,44 @@
+{
+  "properties": {
+    "CLANG_BINARY": {
+      "default": "clang"
+    },
+    "LDAPVC_BINARY": {
+      "default": "ldapvc"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{CLANG_BINARY}",
+      "binproviders": "env,brew",
+      "overrides": {
+        "brew": {
+          "install_args": [
+            "llvm"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{LDAPVC_BINARY}",
+      "binproviders": "env,brew",
+      "overrides": {
+        "env": {
+          "version": [
+            "ldapvc",
+            "-VV"
+          ]
+        },
+        "brew": {
+          "install_args": [
+            "openldap"
+          ],
+          "version": [
+            "ldapvc",
+            "-VV"
+          ]
+        }
+      }
+    }
+  ]
+}

.github/configs/ci-macos-openldap-env.json

@@ -0,0 +1,21 @@
+{
+  "properties": {
+    "LDAPVC_BINARY": {
+      "default": "ldapvc"
+    }
+  },
+  "required_binaries": [
+    {
+      "name": "{LDAPVC_BINARY}",
+      "binproviders": "env",
+      "overrides": {
+        "env": {
+          "version": [
+            "ldapvc",
+            "-VV"
+          ]
+        }
+      }
+    }
+  ]
+}

.github/configs/ci-tooling.json

@@ -0,0 +1,345 @@
+{
+  "properties": {
+    "CURL_BINARY": {
+      "default": "curl"
+    },
+    "BASH_BINARY": {
+      "default": "bash"
+    },
+    "CHMOD_BINARY": {
+      "default": "chmod"
+    },
+    "DOCKER_BINARY": {
+      "default": "docker"
+    },
+    "GIT_BINARY": {
+      "default": "git"
+    },
+    "GREP_BINARY": {
+      "default": "grep"
+    },
+    "GH_BINARY": {
+      "default": "gh"
+    },
+    "EXPORTFS_BINARY": {
+      "default": "exportfs"
+    },
+    "JQ_BINARY": {
+      "default": "jq"
+    },
+    "MKDIR_BINARY": {
+      "default": "mkdir"
+    },
+    "MOUNT_CIFS_BINARY": {
+      "default": "mount.cifs"
+    },
+    "MOUNT_NFS_BINARY": {
+      "default": "mount.nfs"
+    },
+    "NODE_BINARY": {
+      "default": "node"
+    },
+    "PS_BINARY": {
+      "default": "ps"
+    },
+    "PYTHON_BINARY": {
+      "default": "python"
+    },
+    "PV_BINARY": {
+      "default": "pv"
+    },
+    "SSH_BINARY": {
+      "default": "ssh"
+    },
+    "SED_BINARY": {
+      "default": "sed"
+    },
+    "SMBD_BINARY": {
+      "default": "smbd"
+    },
+    "SUDO_BINARY": {
+      "default": "sudo"
+    },
+    "SYSTEMCTL_BINARY": {
+      "default": "systemctl"
+    },
+    "TAR_BINARY": {
+      "default": "tar"
+    },
+    "TREE_BINARY": {
+      "default": "tree"
+    },
+    "TR_BINARY": {
+      "default": "tr"
+    },
+    "RM_BINARY": {
+      "default": "rm"
+    },
+    "UMOUNT_BINARY": {
+      "default": "umount"
+    },
+    "UV_BINARY": {
+      "default": "uv"
+    },
+    "UNAME_BINARY": {
+      "default": "uname"
+    }
+  },
+  "ci_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{JQ_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{UV_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "bsd_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env,pyinfra"
+    }
+  ],
+  "freebsd_binaries": [
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env,pyinfra"
+    }
+  ],
+  "docker_binaries": [
+    {
+      "name": "{DOCKER_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{UNAME_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "deploy_binaries": [
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SSH_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "docker_debug_binaries": [
+    {
+      "name": "{DOCKER_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{PV_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{TAR_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{TREE_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ],
+  "docker_validation_binaries": [
+    {
+      "name": "{DOCKER_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{GREP_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{MKDIR_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{RM_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SED_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SSH_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{TR_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{UNAME_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "mount_validation_binaries": [
+    {
+      "name": "{BASH_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{CHMOD_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{EXPORTFS_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "nfs-kernel-server"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{MKDIR_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{MOUNT_CIFS_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "cifs-utils"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{MOUNT_NFS_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "nfs-common"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{SMBD_BINARY}",
+      "binproviders": "env,apt",
+      "overrides": {
+        "apt": {
+          "install_args": [
+            "samba"
+          ]
+        }
+      }
+    },
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{SYSTEMCTL_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{UMOUNT_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "process_binaries": [
+    {
+      "name": "{PS_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "docs_binaries": [
+    {
+      "name": "{CURL_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{SUDO_BINARY}",
+      "binproviders": "env"
+    }
+  ],
+  "git_binaries": [
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ],
+  "github_binaries": [
+    {
+      "name": "{GH_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ],
+  "node_binaries": [
+    {
+      "name": "{NODE_BINARY}",
+      "binproviders": "env,npm",
+      "min_version": "22.23.1",
+      "overrides": {
+        "npm": {
+          "install_args": [
+            "node@22.23.1"
+          ],
+          "postinstall_scripts": true
+        }
+      }
+    }
+  ],
+  "release_binaries": [
+    {
+      "name": "{UV_BINARY}",
+      "binproviders": "env,pip,brew"
+    },
+    {
+      "name": "{PYTHON_BINARY}",
+      "binproviders": "env"
+    },
+    {
+      "name": "{GH_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{GIT_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{JQ_BINARY}",
+      "binproviders": "env,brew,apt"
+    },
+    {
+      "name": "{CURL_BINARY}",
+      "binproviders": "env,brew,apt"
+    }
+  ]
+}

.github/scripts/discover_test_matrix.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+"""Discover every ArchiveBox test file and expose it as one CI job."""
+
+import json
+from pathlib import Path
+
+
+def main() -> None:
+    root = Path.cwd().resolve()
+    archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
+
+    if not archivebox_tests:
+        raise SystemExit("No ArchiveBox tests discovered")
+
+    matrix: list[dict[str, object]] = []
+    for path in archivebox_tests:
+        test_path = path.relative_to(root).as_posix()
+        matrix.append(
+            {
+                "name": f"main/{path.stem}",
+                "paths": [test_path],
+                "paths_arg": test_path,
+                "extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
+                "count": 1,
+            },
+        )
+
+    discovered_paths = [path for entry in matrix for path in entry["paths"]]
+    if len(discovered_paths) != len(set(discovered_paths)):
+        raise SystemExit("Tests were not discovered exactly once")
+    if sorted(discovered_paths) != [path.relative_to(root).as_posix() for path in archivebox_tests]:
+        raise SystemExit("Discovered test shard coverage does not match test files")
+
+    print(f"Discovered {len(discovered_paths)} test files exactly once across {len(matrix)} jobs")
+    print(json.dumps(matrix, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/ci.yml

@@ -0,0 +1,92 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [dev, main]
+  push:
+    branches: [dev, main]
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: read
+  packages: write
+  security-events: write
+
+concurrency:
+  group: ci-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
+
+jobs:
+  lint:
+    name: Linters
+    uses: ./.github/workflows/lint.yml
+    secrets: inherit
+
+  install-cli-platform:
+    name: Install and CLI platform compatibility
+    uses: ./.github/workflows/test.yml
+    secrets: inherit
+
+  tests:
+    name: Discovered test matrix
+    uses: ./.github/workflows/test-parallel.yml
+    secrets: inherit
+
+  documentation:
+    name: Documentation and root tests
+    uses: ./.github/workflows/docs.yml
+    secrets: inherit
+
+  codeql:
+    name: CodeQL
+    uses: ./.github/workflows/codeql.yml
+    secrets: inherit
+
+  python-artifacts:
+    name: Tested Python artifacts
+    uses: ./.github/workflows/pip.yml
+    secrets: inherit
+
+  docker-digests:
+    name: Tested Docker images
+    uses: ./.github/workflows/docker.yml
+    with:
+      push_digests: ${{ github.event_name == 'push' }}
+    secrets: inherit
+
+  required:
+    name: All required CI lanes
+    if: ${{ always() }}
+    needs:
+      - lint
+      - install-cli-platform
+      - tests
+      - documentation
+      - codeql
+      - python-artifacts
+      - docker-digests
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Verify every required lane succeeded
+        env:
+          LINT_RESULT: ${{ needs.lint.result }}
+          INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
+          TESTS_RESULT: ${{ needs.tests.result }}
+          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
+          CODEQL_RESULT: ${{ needs.codeql.result }}
+          PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
+          DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
+        run: |
+          for result in \
+            "$LINT_RESULT" \
+            "$INSTALL_CLI_PLATFORM_RESULT" \
+            "$TESTS_RESULT" \
+            "$DOCUMENTATION_RESULT" \
+            "$CODEQL_RESULT" \
+            "$PYTHON_ARTIFACTS_RESULT" \
+            "$DOCKER_DIGESTS_RESULT"
+          do
+            [[ "$result" == success ]]
+          done
+          echo 'All required CI lanes and tested release artifacts succeeded.'

.github/workflows/claude.yml

@@ -17,33 +17,24 @@ jobs:
       (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
       (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
       (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       pull-requests: write
       issues: write
       id-token: write
-      actions: read # Required for Claude to read CI results on PRs
+      actions: read
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 1
 
       - name: Run Claude Code
         id: claude
-        uses: anthropics/claude-code-action@v1
+        uses: anthropics/claude-code-action@b76a0776ae74036e77cd11018083743453d7ad35 # v1
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-
-          # This is an optional setting that allows Claude to read CI results on PRs
           additional_permissions: |
             actions: read
-
-          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
-          # prompt: 'Update the pull request description to include a summary of changes.'
-
-          # Optional: Add claude_args to customize behavior and configuration
-          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
-          # or https://code.claude.com/docs/en/cli-reference for available options
           claude_args: '--allowed-tools Bash(gh pr:*)'

.github/workflows/codeql.yml

@@ -1,46 +1,21 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
 name: "CodeQL"
 
 on:
-  push:
-    branches: [ "dev" ]
-  pull_request:
-    branches: [ "dev" ]
+  workflow_call:
   schedule:
     - cron: '33 17 * * 6'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: codeql-${{ github.repository }}-${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   analyze:
     name: Analyze (${{ matrix.language }})
-    # Runner size impacts CodeQL analysis time. To learn more, please see:
-    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
-    #   - https://gh.io/supported-runners-and-hardware-resources
-    #   - https://gh.io/using-larger-runners (GitHub.com only)
-    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
-    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
-    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    runs-on: ubuntu-24.04
     permissions:
-      # required for all workflows
       security-events: write
-
-      # required to fetch internal or private CodeQL packs
       packages: read
-
-      # only required for workflows in private repositories
       actions: read
       contents: read
 
@@ -50,47 +25,18 @@ jobs:
         include:
         - language: python
           build-mode: none
-        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
-        # Use `c-cpp` to analyze code written in C, C++ or both
-        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
-        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
-        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
-        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
-        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
-        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+        - language: javascript-typescript
+          build-mode: none
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-    # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
       with:
         languages: ${{ matrix.language }}
         build-mode: ${{ matrix.build-mode }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-
-        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-    # If the analyze step fails for one of the languages you are analyzing with
-    # "We were unable to automatically build your code", modify the matrix above
-    # to set the build mode to "manual" for that language. Then modify this step
-    # to build your code.
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-    - if: matrix.build-mode == 'manual'
-      run: |
-        echo 'If you are using a "manual" build mode for one or more of the' \
-          'languages you are analyzing, replace this with the commands to build' \
-          'your code, for example:'
-        echo '  make bootstrap'
-        echo '  make release'
-        exit 1
-
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@b7351df727350dca84cb9d725d57dcf5bc82ba26 # v3
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/deploy-publicsite.yml

@@ -6,6 +6,13 @@ on:
       - dev
     paths:
       - publicsite/**
+      - archivebox/**
+      - bin/collect_ui_screenshots.*
+      - bin/generate_ui_screenshot_gallery.py
+      - bin/take_screenshot.js
+      - docs/Screenshots.md
+      - pyproject.toml
+      - uv.lock
       - .github/workflows/deploy-publicsite.yml
   workflow_dispatch:
 
@@ -15,29 +22,42 @@ permissions:
   id-token: write
 
 concurrency:
-  group: github-pages-publicsite
-  cancel-in-progress: true
+  group: ${{ github.event_name == 'workflow_dispatch' && format('github-pages-publicsite-manual-{0}', github.run_id) || 'github-pages-publicsite' }}
+  cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
 
 jobs:
   deploy:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 1
 
+      - name: Setup Python and uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: false
+
+      - name: Install ArchiveBox and screenshot dependencies
+        run: uv sync --no-cache --frozen
+
+      - name: Build UI screenshot gallery
+        run: bin/collect_ui_screenshots.sh
+        env:
+          BASE_URL: http://archivebox.localhost:8000
+
       - name: Setup Pages
-        uses: actions/configure-pages@v5
+        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
 
       - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
+        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
         with:
           path: ./publicsite
 
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4

.github/workflows/docker.yml

@@ -1,28 +1,23 @@
 name: Build Docker image
 
 on:
-  workflow_dispatch:
   workflow_call:
-  push:
-    branches:
-      - "**"
-    tags:
-      - 'v*'
-  # pull_request:
+    inputs:
+      push_digests:
+        description: Push validated per-platform images by digest for the release workflow
+        required: false
+        default: false
+        type: boolean
 
 env:
   DOCKERHUB_IMAGE: archivebox/archivebox
   GHCR_IMAGE: ghcr.io/archivebox/archivebox
-  ABX_DL_IMAGE: archivebox/abx-dl:latest
 
 permissions:
+  actions: read
   contents: read
   packages: write
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   build:
     name: build ${{ matrix.platform }}
@@ -35,60 +30,81 @@ jobs:
             runner: ubuntu-24.04
             cache_scope: docker-amd64
             artifact_name: digest-linux-amd64
+            local_tag: archivebox-ci:amd64
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
             cache_scope: docker-arm64
             artifact_name: digest-linux-arm64
+            local_tag: archivebox-ci:arm64
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
-        # with:
-        #   submodules: true
-        #   fetch-depth: 1
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Wait for released ArchiveBox deps on PyPI
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Resolve Docker through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        shell: bash
         run: |
-          python3 - <<'PY'
-          import json
-          import re
-          import sys
-          import time
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python - <<'PY'
           import tomllib
-          import urllib.request
+          from pathlib import Path
 
-          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
-          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
-          required = {}
-          for dep in deps:
-              for name in watched:
-                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
-                  if match:
-                      required[name] = match.group(2)
-
-          deadline = time.monotonic() + 300
-          missing = required.copy()
-          while missing and time.monotonic() < deadline:
-              for name, version in list(missing.items()):
-                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
-                      releases = json.load(resp)["releases"]
-                  if version in releases:
-                      print(f"{name} {version} is available on PyPI")
-                      missing.pop(name)
-                  else:
-                      print(f"{name} {version} is not available on PyPI yet")
-              if missing:
-                  time.sleep(10)
-
-          if missing:
-              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
-              sys.exit(1)
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          matches = [package["version"] for package in packages if package["name"] == "abxpkg"]
+          if len(matches) != 1:
+              raise SystemExit(f"Expected one locked abxpkg package, found {len(matches)}")
+          print(matches[0])
           PY
+          )"
+          test -n "$ABXPKG_VERSION"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+          echo "ABXPKG_VERSION=$ABXPKG_VERSION" >> "$GITHUB_ENV"
+          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+
+          docker_env="$(
+            uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+              --install \
+              --json \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries"
+          )"
+          JQ_BINARY="$ABXPKG_LIB_DIR/env/bin/jq"
+          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
+            <<< "$docker_env" >> "$GITHUB_ENV"
+
+          docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
+          UV_BINARY="$ABXPKG_LIB_DIR/env/bin/uv"
+          {
+            echo "DOCKER_BINARY=$docker_binary"
+            echo "JQ_BINARY=$JQ_BINARY"
+            echo "UV_BINARY=$UV_BINARY"
+          } >> "$GITHUB_ENV"
+          for binary in "$docker_binary" "$JQ_BINARY" "$UV_BINARY"; do
+            test -L "$binary"
+            test -x "$binary"
+          done
+          "$docker_binary" --version
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
         with:
-          version: latest
           install: true
 
       - name: Builder instance name
@@ -97,33 +113,50 @@ jobs:
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
-      - name: Wait for published abx-dl image
+      - name: Verify published abx-dl image
         id: abx_dl_image
         shell: bash
         run: |
           set -Eeuo pipefail
-          deadline=$((SECONDS + 1800))
-          until docker buildx imagetools inspect "${ABX_DL_IMAGE}" >/tmp/abx-dl-image.json; do
-            if (( SECONDS >= deadline )); then
-              echo "Timed out waiting for published ${ABX_DL_IMAGE}" >&2
-              exit 1
-            fi
-            echo "${ABX_DL_IMAGE} is not published yet; waiting..."
-            sleep 30
-          done
+          ABX_DL_VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
+          import tomllib
+          from pathlib import Path
 
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          matches = [package["version"] for package in packages if package["name"] == "abx-dl"]
+          if len(matches) != 1:
+              raise SystemExit(f"Expected one locked abx-dl package, found {len(matches)}")
+          print(matches[0])
+          PY
+          )"
+          test -n "$ABX_DL_VERSION"
+          ABX_DL_REFERENCE="${ABX_DL_IMAGE:-archivebox/abx-dl:${ABX_DL_VERSION}}"
+          INSPECT_OUTPUT="$("$DOCKER_BINARY" buildx imagetools inspect "$ABX_DL_REFERENCE")"
+          printf '%s\n' "$INSPECT_OUTPUT"
+          ABX_DL_DIGEST="$(INSPECT_OUTPUT="$INSPECT_OUTPUT" "$UV_BINARY" run --no-cache --no-project python - <<'PY'
+          import os
+
+          digest = next(
+              (fields[1] for line in os.environ["INSPECT_OUTPUT"].splitlines() if (fields := line.split()) and fields[0] == "Digest:"),
+              "",
+          )
+          print(digest)
+          PY
+          )"
+          [[ "$ABX_DL_DIGEST" == sha256:* ]]
+          ABX_DL_IMAGE="${ABX_DL_REFERENCE%%@*}@${ABX_DL_DIGEST}"
           echo "image=${ABX_DL_IMAGE}" >> "$GITHUB_OUTPUT"
-          docker buildx imagetools inspect "${ABX_DL_IMAGE}"
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        if: inputs.push_digests
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        if: inputs.push_digests
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -134,13 +167,14 @@ jobs:
         shell: bash
         run: |
           set -Eeuo pipefail
-          VERSION="$(python3 - <<'PY'
+          VERSION="$("$UV_BINARY" run --no-cache --no-project python - <<'PY'
           import tomllib
+          from pathlib import Path
 
-          with open("pyproject.toml", "rb") as f:
-              print(tomllib.load(f)["project"]["version"])
+          print(tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"])
           PY
           )"
+          test -n "$VERSION"
 
           {
             echo 'labels<<EOF'
@@ -156,27 +190,274 @@ jobs:
 
       - name: Build and push digest
         id: docker_build
-        uses: docker/build-push-action@v6
+        if: inputs.push_digests
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
         with:
           context: ./
           file: ./Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: |
-            ${{ env.DOCKERHUB_IMAGE }}
-            ${{ env.GHCR_IMAGE }}
           labels: ${{ steps.docker_meta.outputs.labels }}
           build-args: |
             ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
-          cache-from: type=gha,scope=${{ matrix.cache_scope }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.cache_scope }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+          pull: true
+          platforms: ${{ matrix.platform }}
+          outputs: |
+            type=docker,name=${{ matrix.local_tag }}
+            type=oci,dest=${{ runner.temp }}/archivebox-image.tar
+            type=image,"name=${{ env.DOCKERHUB_IMAGE }},${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true
+
+      - name: Build pull request image
+        if: ${{ !inputs.push_digests }}
+        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
+        with:
+          context: ./
+          file: ./Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          load: true
+          tags: ${{ matrix.local_tag }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          build-args: |
+            ABX_DL_IMAGE=${{ steps.abx_dl_image.outputs.image }}
+            ARCHIVEBOX_COMMIT_HASH=${{ github.sha }}
+          pull: true
           platforms: ${{ matrix.platform }}
-          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
 
       - name: Image digest
+        if: inputs.push_digests
         run: echo ${{ steps.docker_build.outputs.digest }}
 
+      - name: Validate exact built image version and commit
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          IMAGE="${{ matrix.local_tag }}"
+          SHORT_SHA="${GITHUB_SHA::7}"
+
+          DATA_DIR="$(mktemp -d)"
+          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox init
+          ARCHIVEBOX_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox version)"
+          printf '%s\n' "$ARCHIVEBOX_VERSION_OUTPUT"
+          "$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' -v "$DATA_DIR":/data "$IMAGE" archivebox status
+          IMAGE_VERSION_OUTPUT="$("$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' --entrypoint cat "$IMAGE" /VERSION.txt)"
+          ARCHIVEBOX_VERSION_OUTPUT="$ARCHIVEBOX_VERSION_OUTPUT" IMAGE_VERSION_OUTPUT="$IMAGE_VERSION_OUTPUT" \
+            "$UV_BINARY" run --no-cache --no-project python - \
+            "ArchiveBox v${{ steps.docker_meta.outputs.version }}" \
+            "COMMIT_HASH=${SHORT_SHA}" \
+            "COMMIT_HASH=${GITHUB_SHA}" <<'PY'
+          import os
+          import sys
+
+          version_output = os.environ["ARCHIVEBOX_VERSION_OUTPUT"]
+          image_version_lines = os.environ["IMAGE_VERSION_OUTPUT"].splitlines()
+          if sys.argv[1] not in version_output:
+              raise SystemExit(f"Missing version marker: {sys.argv[1]}")
+          if sys.argv[2] not in version_output:
+              raise SystemExit(f"Missing short commit marker: {sys.argv[2]}")
+          if sys.argv[3] not in image_version_lines:
+              raise SystemExit(f"Missing exact commit marker: {sys.argv[3]}")
+          PY
+
+          "$DOCKER_BINARY" run --rm --user 911:911 --platform '${{ matrix.platform }}' --entrypoint /venv/bin/python3 "$IMAGE" -c \
+            'import json, pathlib, stat; cache_file = "/opt/archivebox/lib/env/derived.env"; raw = open(cache_file).read().strip(); records = json.loads(raw[len("ABXPKG_DERIVED_CACHE=\x27"):-1]); plans = [plan for record in records.values() for plan in record.get("script_exec_plans", {}).values()]; contexts = [json.loads(plan["run_context"])["base"]["abxpkg_env"] for plan in plans]; fingerprints = [fingerprint for plan in plans for fingerprint in plan["fingerprint"]]; actual = lambda fingerprint: (lambda path, value: {"path": str(path.resolve(strict=False)), "size": value.st_size, "mtime_ns": value.st_mtime_ns, "mode": stat.S_IMODE(value.st_mode), "euid": value.st_uid})(pathlib.Path(fingerprint["path"]), pathlib.Path(fingerprint["path"]).stat()); mismatches = [(fingerprint, actual(fingerprint)) for fingerprint in fingerprints if fingerprint != actual(fingerprint)]; assert plans and {plan.get("euid") for plan in plans} == {911}; assert all("ABXPKG_TMP_CACHE_DIR" not in context for context in contexts); assert not mismatches, mismatches[:3]'
+
+      - name: Resolve real Docker CLI test dependencies through abxpkg
+        if: matrix.platform == 'linux/amd64'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mount_env="$(
+            uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+              --install \
+              --json \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:mount_validation_binaries"
+          )"
+          "$JQ_BINARY" -r 'to_entries[] | "\(.key)=\(.value)"' \
+            <<< "$mount_env" >> "$GITHUB_ENV"
+
+          for binary_name in bash chmod curl exportfs mkdir mount.cifs mount.nfs smbd sudo systemctl umount; do
+            binary="$ABXPKG_LIB_DIR/env/bin/$binary_name"
+            test -L "$binary"
+            test -x "$binary"
+          done
+
+      - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
+        if: matrix.platform == 'linux/amd64'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          NFS_SOURCE="/srv/archivebox-ci-nfs"
+          SMB_SOURCE="/srv/archivebox-ci-smb"
+          NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
+          SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
+          mounted_nfs=0
+          mounted_smb=0
+
+          cleanup() {
+            local original_status=$? cleanup_status=0
+            trap - EXIT
+            if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
+              cleanup_status=1
+            fi
+            if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
+              cleanup_status=1
+            fi
+            if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
+              cleanup_status=1
+            fi
+            if [[ "$original_status" != "0" ]]; then
+              exit "$original_status"
+            fi
+            exit "$cleanup_status"
+          }
+          trap cleanup EXIT
+
+          "$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+          "$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
+          "$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
+          "$SUDO_BINARY" "$BASH_BINARY" -c \
+            "printf '%s *(rw,sync,no_subtree_check,no_root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
+          "$SUDO_BINARY" "$BASH_BINARY" -c \
+            "printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
+
+          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
+          "$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
+          "$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
+          "$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
+          "$SYSTEMCTL_BINARY" is-active --quiet smbd.service
+
+          "$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
+          mounted_nfs=1
+          "$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
+            -o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
+          mounted_smb=1
+
+          NFS_TEST_DIR="$NFS_MOUNT" \
+          SMB_TEST_DIR="$SMB_MOUNT" \
+          IMAGE="${{ matrix.local_tag }}" \
+            "$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
+
+      - name: Validate Docker CLI add/list and compose behavior
+        if: matrix.platform == 'linux/amd64'
+        shell: bash
+        env:
+          ARCHIVEBOX_IMAGE: ${{ matrix.local_tag }}
+        run: |
+          set -Eeuo pipefail
+          "$MKDIR_BINARY" "${{ github.workspace }}/data"
+          "$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
+
+          "$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          "$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
+
+          "$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
+          echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
+
+          archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
+          [[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
+          [[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
+
+          "$DOCKER_BINARY" compose run --rm archivebox init
+          "$DOCKER_BINARY" compose up -d --wait
+          "$DOCKER_BINARY" network connect archivebox_default www-nginx
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
+          admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js')"
+          [[ "$index_html" == *ArchiveBox* ]]
+          [[ "$admin_js" == *window.django* ]]
+
+          "$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
+          index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:8000')"
+          [[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
+          "$DOCKER_BINARY" compose down
+
+      - name: Validate compressed image size
+        if: inputs.push_digests
+        env:
+          BUILD_METADATA: ${{ steps.docker_build.outputs.metadata }}
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          LOCAL_IMAGE_ID="$("$DOCKER_BINARY" image inspect '${{ matrix.local_tag }}' --format '{{.Id}}')"
+          "$UV_BINARY" run --no-cache --no-project python - \
+            '${{ runner.temp }}/archivebox-image.tar' \
+            '${{ steps.docker_build.outputs.digest }}' \
+            "$LOCAL_IMAGE_ID" \
+            "$((820 * 1024 * 1024))" <<'PY'
+          from hashlib import sha256
+          import json
+          import os
+          import sys
+          import tarfile
+
+          archive_path, pushed_digest, local_image_id, limit_text = sys.argv[1:]
+          limit = int(limit_text)
+          metadata = json.loads(os.environ["BUILD_METADATA"])
+          if metadata["containerimage.digest"] != pushed_digest:
+              raise SystemExit("Build push transaction digest does not match the action output")
+          if not pushed_digest.startswith("sha256:") or len(pushed_digest) != 71:
+              raise SystemExit(f"Invalid pushed digest: {pushed_digest}")
+
+          with tarfile.open(archive_path) as archive:
+              def read_blob(descriptor):
+                  algorithm, digest = descriptor["digest"].split(":", 1)
+                  if algorithm != "sha256":
+                      raise SystemExit(f"Unsupported OCI digest: {descriptor['digest']}")
+                  member = archive.extractfile(f"blobs/sha256/{digest}")
+                  if member is None:
+                      raise SystemExit(f"Missing OCI blob: {descriptor['digest']}")
+                  blob = member.read()
+                  if len(blob) != descriptor["size"]:
+                      raise SystemExit(f"OCI size mismatch: {descriptor['digest']}")
+                  if sha256(blob).hexdigest() != digest:
+                      raise SystemExit(f"OCI digest mismatch: {descriptor['digest']}")
+                  return blob
+
+              index_member = archive.extractfile("index.json")
+              if index_member is None:
+                  raise SystemExit("OCI export is missing index.json")
+              index = json.load(index_member)
+
+              manifests = []
+
+              def collect(descriptor):
+                  document = json.loads(read_blob(descriptor))
+                  media_type = descriptor["mediaType"]
+                  if "image.index" in media_type or "manifest.list" in media_type:
+                      for child in document["manifests"]:
+                          collect(child)
+                  elif "image.manifest" in media_type:
+                      config_type = document["config"]["mediaType"]
+                      if "image.config" in config_type or "container.image" in config_type:
+                          manifests.append(document)
+
+              for root in index["manifests"]:
+                  collect(root)
+              if len(manifests) != 1:
+                  raise SystemExit(f"Expected one runnable image manifest, found {len(manifests)}")
+
+              manifest = manifests[0]
+              config_digest = manifest["config"]["digest"]
+              if metadata["containerimage.config.digest"] != config_digest:
+                  raise SystemExit("OCI config digest does not match the pushed build result")
+              if local_image_id != config_digest:
+                  raise SystemExit("Locally tested image does not match the OCI build result")
+              descriptors = [manifest["config"], *manifest["layers"]]
+              for descriptor in descriptors:
+                  read_blob(descriptor)
+              total = sum(descriptor["size"] for descriptor in descriptors)
+
+          print(
+              f"{pushed_digest} compressed_size={total // 1024 // 1024} MiB "
+              f"limit={limit // 1024 // 1024} MiB",
+          )
+          if total > limit:
+              raise SystemExit(f"{pushed_digest} is over the compressed size limit")
+          PY
+
       - name: Export digest
+        if: inputs.push_digests
         shell: bash
         run: |
           set -Eeuo pipefail
@@ -185,147 +466,10 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@v4
+        if: inputs.push_digests
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: ${{ matrix.artifact_name }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
-
-  publish:
-    name: publish multiarch tags
-    runs-on: ubuntu-24.04
-    needs:
-      - build
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          version: latest
-          install: true
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: digest-*
-          merge-multiple: true
-
-      - name: Collect Docker tags
-        id: docker_meta
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          VERSION="$(python3 - <<'PY'
-          import tomllib
-
-          with open("pyproject.toml", "rb") as f:
-              print(tomllib.load(f)["project"]["version"])
-          PY
-          )"
-          BRANCH_TAG="$(printf '%s' "${GITHUB_REF_NAME}" | tr -c 'A-Za-z0-9_.-' '-' | sed -E 's/^-+//; s/-+$//; s/-+/-/g' | cut -c1-128)"
-          SHORT_SHA="${GITHUB_SHA::12}"
-          test -n "$BRANCH_TAG"
-          test -n "$SHORT_SHA"
-
-          {
-            echo 'dockerhub_tags<<EOF'
-            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-              echo "${DOCKERHUB_IMAGE}:latest"
-            fi
-            echo "${DOCKERHUB_IMAGE}:${BRANCH_TAG}"
-            echo "${DOCKERHUB_IMAGE}:${VERSION}"
-            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-            echo 'EOF'
-            echo 'ghcr_tags<<EOF'
-            if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-              echo "${GHCR_IMAGE}:latest"
-            fi
-            echo "${GHCR_IMAGE}:${BRANCH_TAG}"
-            echo "${GHCR_IMAGE}:${VERSION}"
-            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-            echo 'EOF'
-            echo "version=${VERSION}"
-            echo "branch_tag=${BRANCH_TAG}"
-            echo "short_sha=${SHORT_SHA}"
-          } >> "$GITHUB_OUTPUT"
-
-          echo "[+] Publishing Docker Hub tags:"
-          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-            printf '%s\n' "${DOCKERHUB_IMAGE}:latest"
-          fi
-          printf '%s\n' "${DOCKERHUB_IMAGE}:${BRANCH_TAG}" "${DOCKERHUB_IMAGE}:${VERSION}" "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
-          echo "[+] Publishing GHCR tags:"
-          if [[ "${GITHUB_REF_NAME}" == "main" ]]; then
-            printf '%s\n' "${GHCR_IMAGE}:latest"
-          fi
-          printf '%s\n' "${GHCR_IMAGE}:${BRANCH_TAG}" "${GHCR_IMAGE}:${VERSION}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"
-
-      - name: Create Docker Hub manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
-          [[ "${#DIGESTS[@]}" -gt 0 ]]
-
-          TAG_ARGS=()
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
-          done <<< "${{ steps.docker_meta.outputs.dockerhub_tags }}"
-
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do
-            REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}")
-          done
-
-          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Create GHCR manifest
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          mapfile -t DIGESTS < <(find /tmp/digests -maxdepth 1 -type f -printf '%f\n' | sort)
-          [[ "${#DIGESTS[@]}" -gt 0 ]]
-
-          TAG_ARGS=()
-          while IFS= read -r tag; do
-            [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag")
-          done <<< "${{ steps.docker_meta.outputs.ghcr_tags }}"
-
-          REFS=()
-          for digest in "${DIGESTS[@]}"; do
-            REFS+=("${GHCR_IMAGE}@sha256:${digest}")
-          done
-
-          docker buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
-
-      - name: Inspect published images
-        shell: bash
-        run: |
-          set -Eeuo pipefail
-          docker buildx imagetools inspect "${DOCKERHUB_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-          docker buildx imagetools inspect "${GHCR_IMAGE}:sha-${{ steps.docker_meta.outputs.short_sha }}"
-
-      - name: Update README
-        uses: peter-evans/dockerhub-description@v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          repository: archivebox/archivebox

.github/workflows/docs.yml

@@ -0,0 +1,156 @@
+name: Documentation tests
+
+on:
+  workflow_call:
+
+env:
+  PYTHONIOENCODING: utf-8
+  USE_COLOR: "False"
+  SHOW_PROGRESS: "False"
+
+jobs:
+  discover:
+    name: Discover documentation examples
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    outputs:
+      matrix: ${{ steps.inventory.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+      - name: Install ArchiveBox development dependencies
+        run: uv sync --no-cache --locked --dev
+      - name: Resolve documentation tools through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
+        run: |
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          uv run --no-cache --no-sync abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/docs/mermaid-binary.json:required_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/docs/nginx-binary.json:required_binaries"
+          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
+          mmdc_binary="$ABXPKG_LIB_DIR/env/bin/mmdc"
+          nginx_binary="$ABXPKG_LIB_DIR/env/bin/nginx"
+          test -L "$bash_binary"
+          test -L "$mmdc_binary"
+          test -L "$nginx_binary"
+          test -x "$bash_binary"
+          test -x "$mmdc_binary"
+          test -x "$nginx_binary"
+          {
+            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
+            echo "BASH_BINARY=$bash_binary"
+          } >> "$GITHUB_ENV"
+      - id: inventory
+        name: Validate the complete authored documentation inventory
+        run: |
+          matrix="$(uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py matrix)"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+
+  examples:
+    name: docs/${{ matrix.name }}
+    needs: discover
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          submodules: true
+          fetch-depth: 1
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        with:
+          python-version: "3.13"
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+      - name: Install ArchiveBox
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        run: uv sync --no-cache --locked --dev
+      - name: Resolve documentation runtime through abxpkg
+        if: ${{ !contains(fromJSON('["freebsd", "openbsd"]'), matrix.environment) }}
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/archivebox-docs-lib
+        run: |
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          dependency_sources=(
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
+          )
+          if [[ '${{ matrix.environment }}' != macos ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
+          fi
+          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
+          elif [[ '${{ matrix.environment }}' == docker ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
+          elif [[ '${{ matrix.environment }}' == macos ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")
+          fi
+          uv run --no-cache --no-sync abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            "${dependency_sources[@]}"
+          bash_binary="$ABXPKG_LIB_DIR/env/bin/bash"
+          python_binary="$ABXPKG_LIB_DIR/env/bin/python"
+          uv_binary="$ABXPKG_LIB_DIR/env/bin/uv"
+          test -L "$bash_binary"
+          test -x "$bash_binary"
+          test -L "$python_binary"
+          test -x "$python_binary"
+          test -L "$uv_binary"
+          test -x "$uv_binary"
+          {
+            echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR"
+            echo "BASH_BINARY=$bash_binary"
+            echo "PYTHON_BINARY=$python_binary"
+            echo "UV_BINARY=$uv_binary"
+          } >> "$GITHUB_ENV"
+          if [[ '${{ matrix.environment }}' != macos ]]; then
+            sudo_binary="$ABXPKG_LIB_DIR/env/bin/sudo"
+            test -L "$sudo_binary"
+            test -x "$sudo_binary"
+            echo "SUDO_BINARY=$sudo_binary" >> "$GITHUB_ENV"
+          fi
+          if [[ '${{ matrix.environment }}' != macos ]]; then
+            echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+          fi
+          if [[ '${{ matrix.environment }}' == docker ]]; then
+            docker_binary="$ABXPKG_LIB_DIR/env/bin/docker"
+            test -L "$docker_binary"
+            test -x "$docker_binary"
+            echo "DOCKER_BINARY=$docker_binary" >> "$GITHUB_ENV"
+          elif [[ '${{ matrix.environment }}' == macos ]]; then
+            test -L "$ABXPKG_LIB_DIR/env/bin/brew"
+            test -x "$ABXPKG_LIB_DIR/env/bin/brew"
+          fi
+      - name: Build the exact checkout's documentation image
+        if: ${{ matrix.environment == 'docker' }}
+        run: '"$DOCKER_BINARY" build --tag archivebox/archivebox:dev .'
+      - name: Run the Ubuntu, Docker, or macOS documentation lane
+        if: ${{ !contains(fromJSON('["root", "freebsd", "openbsd"]'), matrix.environment) }}
+        run: uv run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Run the root documentation lane
+        if: ${{ matrix.environment == 'root' }}
+        run: |
+          "$SUDO_BINARY" \
+            --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
+            "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'

.github/workflows/duplicate-issue-detection.yml

@@ -6,18 +6,55 @@ on:
 
 jobs:
   check-duplicates:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       issues: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           fetch-depth: 1
 
-      - name: Install opencode
-        run: curl -fsSL https://opencode.ai/install | bash
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+
+      - name: Resolve OpenCode and workflow tools through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        run: |
+          set -Eeuo pipefail
+          uv sync --no-cache --locked --no-sources --no-install-project
+          OPENCODE_CONFIG="$(uv run --no-cache --no-sync --no-sources python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "opencode" / "config.json")')"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          export PATH="$ABXPKG_LIB_DIR/env/bin:$PATH"
+          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
+          echo "$ABXPKG_LIB_DIR/env/bin" >> "$GITHUB_PATH"
+          opencode_env="$(
+            uv run --no-cache --no-sync --no-sources abxpkg env \
+              --install \
+              --json \
+              --lib="$ABXPKG_LIB_DIR" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries" \
+              --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:github_binaries" \
+              --deps-from="$OPENCODE_CONFIG:required_binaries"
+          )"
+          jq_binary="$ABXPKG_LIB_DIR/env/bin/jq"
+          "$jq_binary" -r 'to_entries[] | "\(.key)=\(.value)"' \
+            <<< "$opencode_env" >> "$GITHUB_ENV"
+
+          opencode_binary="$ABXPKG_LIB_DIR/env/bin/opencode"
+          echo "OPENCODE_BINARY=$opencode_binary" >> "$GITHUB_ENV"
+          test -L "$opencode_binary"
+          test -x "$opencode_binary"
 
       - name: Check for duplicate issues
         env:
@@ -33,7 +70,7 @@ jobs:
               "webfetch": "allow"
             }
         run: |
-          opencode run -m anthropic/claude-haiku-4-5 "A new issue has been created:
+          "$OPENCODE_BINARY" run -m anthropic/claude-haiku-4-5 "A new issue has been created:
 
           Issue number:
           ${{ github.event.issue.number }}

.github/workflows/lint.yml

@@ -1,50 +1,38 @@
 name: Run linters
 
 on:
-  workflow_dispatch:
-  push:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   UV_NO_SOURCES: "1"
-  PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
 jobs:
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: "3.13"
           architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
+          version: "0.11.3"
           enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Verify checked dependency lock
+        run: uv lock --no-cache --check
 
       - name: Install dependencies with uv
-        run: |
-          uv venv
-          uv pip install --all-groups -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[sonic,debug]"
+        run: uv sync --no-cache --locked --dev --extra sonic --extra debug
 
       - name: Run prek
-        run: uv run --no-sync --no-sources prek run --all-files
+        run: uv run --no-cache --no-sync --no-sources prek run --all-files

.github/workflows/pip.yml

@@ -1,108 +1,221 @@
 name: Build Pip package
 
 on:
-  workflow_dispatch:
   workflow_call:
-  push:
-    branches:
-      - '**'
-    tags:
-      - 'v*'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
 
 env:
-  PYTHON_VERSION: "3.13"
+  UV_VERSION: "0.11.3"
 
 jobs:
   build:
-    permissions:
-      id-token: write
-
+    name: build distributions
     runs-on: ubuntu-24.04
-    environment: pypi
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
+      - name: Set up Python 3.13
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
-          enable-cache: false
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "3.13"
           architecture: x64
 
-      - name: APT install archivebox dev + run dependencies
-        uses: awalsh128/cache-apt-pkgs-action@latest
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
+          version: ${{ env.UV_VERSION }}
+          enable-cache: false
+          cache-dependency-glob: uv.lock
 
-      - name: Wait for released ArchiveBox deps on PyPI
+      - name: Build every distribution
         run: |
-          python - <<'PY'
-          import json
-          import re
-          import sys
-          import time
-          import tomllib
-          import urllib.request
+          set -Eeuo pipefail
+          uv build --no-cache --all-packages --no-sources --out-dir dist --clear
 
-          watched = {"abxbus", "abxpkg", "abx-plugins", "abx-dl"}
-          deps = tomllib.loads(open("pyproject.toml", "rb").read().decode())["project"]["dependencies"]
-          required = {}
-          for dep in deps:
-              for name in watched:
-                  match = re.match(rf"{re.escape(name)}\s*(==|>=)\s*([^,;\s]+)", dep)
-                  if match:
-                      required[name] = match.group(2)
+          shopt -s nullglob
+          artifacts=(dist/*)
+          wheels=(dist/archivebox-*.whl)
+          sdists=(dist/archivebox-*.tar.gz)
+          [[ "${#artifacts[@]}" -gt 0 ]]
+          [[ "${#wheels[@]}" -eq 1 ]]
+          [[ "${#sdists[@]}" -eq 1 ]]
+          printf '%s\n' "$GITHUB_SHA" > dist/COMMIT_SHA
+          uv run --no-cache --no-project python - <<'PY'
+          import hashlib
+          from pathlib import Path
 
-          deadline = time.monotonic() + 300
-          missing = required.copy()
-          while missing and time.monotonic() < deadline:
-              for name, version in list(missing.items()):
-                  with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/json", timeout=20) as resp:
-                      releases = json.load(resp)["releases"]
-                  if version in releases:
-                      print(f"{name} {version} is available on PyPI")
-                      missing.pop(name)
-                  else:
-                      print(f"{name} {version} is not available on PyPI yet")
-              if missing:
-                  time.sleep(10)
-
-          if missing:
-              print(f"Missing PyPI releases after wait: {missing}", file=sys.stderr)
-              sys.exit(1)
+          root = Path("dist")
+          wheels = list(root.glob("archivebox-*.whl"))
+          sdists = list(root.glob("archivebox-*.tar.gz"))
+          if len(wheels) != 1 or len(sdists) != 1:
+              raise SystemExit("Expected exactly one ArchiveBox wheel and one sdist")
+          artifacts = [root / "COMMIT_SHA", wheels[0], sdists[0]]
+          checksum_lines = []
+          for artifact in artifacts:
+              digest = hashlib.sha256(artifact.read_bytes()).hexdigest()
+              line = f"{digest}  {artifact.name}"
+              checksum_lines.append(line)
+              print(line)
+          Path("dist/SHA256SUMS").write_text("\n".join(checksum_lines) + "\n")
           PY
 
-      - name: UV install archivebox dev + run sub-dependencies
-        run: uv sync --all-extras --no-install-project --no-install-workspace --no-sources --no-cache
-
-      - name: UV build archivebox and archivebox/pkgs/* packages
-        run: |
-          uv build --all
-
-      - name: Publish new package wheels and sdists to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        # ignore when publish to PyPI fails due to duplicate tag
-        continue-on-error: true
+      - name: Upload every distribution
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          password: ${{ secrets.PYPI_PAT_SECRET }}
+          name: python-distributions
+          path: dist/*
+          if-no-files-found: error
+          retention-days: 7
 
-      - name: UV install archivebox and archivebox/pkgs/* locally for tests
-        run: uv sync --all-extras --no-sources --no-cache
+  install-smoke:
+    name: install ${{ matrix.artifact }} / Python ${{ matrix.python }} / ${{ matrix.os }}
+    needs: build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, macos-15]
+        python: ["3.13", "3.14"]
+        artifact: [wheel, sdist]
 
-      - name: UV run archivebox init + archivebox version
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: ${{ env.UV_VERSION }}
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Download distributions
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: python-distributions
+          path: ${{ runner.temp }}/python-distributions
+
+      - name: Install and smoke-test the built ${{ matrix.artifact }}
+        shell: bash
         run: |
-          mkdir -p data && cd data
-          uv run --no-sync --no-sources archivebox init \
-          && uv run --no-sync --no-sources archivebox version
-          # && uv run archivebox add 'https://example.com' \
-          # && uv run archivebox status \
-          # || (echo "UV Failed to run archivebox!" && exit 1)
+          set -Eeuo pipefail
+          shopt -s nullglob
+
+          case "${{ matrix.artifact }}" in
+            wheel) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.whl) ;;
+            sdist) candidates=("$RUNNER_TEMP"/python-distributions/archivebox-*.tar.gz) ;;
+            *) exit 2 ;;
+          esac
+          [[ "${#candidates[@]}" -eq 1 ]]
+          artifact="${candidates[0]}"
+
+          smoke_root="$(mktemp -d "$RUNNER_TEMP/archivebox-package-smoke.XXXXXX")"
+          trap 'rm -rf "$smoke_root"' EXIT
+          smoke_env="$smoke_root/env"
+          data_dir="$smoke_root/data"
+          mkdir -p "$data_dir"
+
+          dependency_artifacts=()
+          while IFS= read -r dependency_artifact; do
+            dependency_artifacts+=("$dependency_artifact")
+          done < <(uv run --no-cache --no-project python - <<'PY'
+          import tomllib
+
+          locked = {
+              package["name"]: package
+              for package in tomllib.load(open("uv.lock", "rb"))["package"]
+          }
+          for package_name in ("abx-dl", "abx-plugins", "abxpkg", "abxbus"):
+              package = locked[package_name]
+              wheels = package.get("wheels") or []
+              if len(wheels) != 1:
+                  raise SystemExit(f"Expected exactly one locked wheel for {package_name}")
+              print(wheels[0]["url"])
+          PY
+          )
+
+          uv venv --python "${{ matrix.python }}" "$smoke_env"
+          uv pip install --no-cache \
+            --python "$smoke_env" \
+            "${dependency_artifacts[@]}" \
+            "$artifact"
+
+          unset PYTHONPATH
+          cd "$data_dir"
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync python - <<'PY'
+          import os
+          from pathlib import Path
+
+          import archivebox
+
+          installed_path = Path(archivebox.__file__).resolve()
+          environment_path = Path(os.environ["VIRTUAL_ENV"]).resolve()
+          workspace_path = Path(os.environ["GITHUB_WORKSPACE"]).resolve()
+          assert installed_path.is_relative_to(environment_path), (installed_path, environment_path)
+          assert not installed_path.is_relative_to(workspace_path), (installed_path, workspace_path)
+          print(installed_path)
+          PY
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox version
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox init
+          VIRTUAL_ENV="$smoke_env" uv run --no-cache --active --no-project --no-sync archivebox status
+
+          if [[ "${{ runner.os }}" == "Linux" ]]; then
+            root_tool_dir="/root/archivebox-package-smoke-${{ matrix.artifact }}-${{ matrix.python }}"
+            uv_binary="$(command -v uv)"
+            sudo rm -rf "$root_tool_dir"
+            sudo mkdir -p "$root_tool_dir/data"
+
+            root_install_args=()
+            for dependency_artifact in "${dependency_artifacts[@]}"; do
+              root_install_args+=(--with "$dependency_artifact")
+            done
+            sudo UV_TOOL_DIR="$root_tool_dir/tools" UV_TOOL_BIN_DIR="$root_tool_dir/bin" \
+              "$uv_binary" tool install --no-cache --python "${{ matrix.python }}" \
+              "${root_install_args[@]}" "$artifact"
+
+            sudo useradd --system --create-home --home-dir /var/lib/archivebox --shell /bin/bash archivebox 2>/dev/null || true
+            sudo rm -rf /var/lib/archivebox
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data' && archivebox init"
+            sudo stat -c %U /var/lib/archivebox | grep -qx archivebox
+            sudo stat -c %U "$root_tool_dir/data" | grep -qx archivebox
+            if sudo find "$root_tool_dir/data" -xdev -user root -print -quit | grep -q .; then
+              echo "Root-owned files remain in $root_tool_dir/data" >&2
+              exit 1
+            fi
+
+            sudo mkdir "$root_tool_dir/external"
+            sudo ln -s "$root_tool_dir/external" "$root_tool_dir/data/external-link"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data' && archivebox status"
+            sudo stat -c %U "$root_tool_dir/external" | grep -qx root
+
+            sudo mkdir "$root_tool_dir/data-universal-init"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data-universal-init' && archivebox status --init"
+            sudo stat -c %U "$root_tool_dir/data-universal-init" | grep -qx archivebox
+
+            sudo mkdir "$root_tool_dir/data-install"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data-install' && archivebox install --dry-run"
+            sudo stat -c %U "$root_tool_dir/data-install" | grep -qx archivebox
+
+            sudo useradd --create-home archivebox-smoke-user
+            smoke_user_group="$(id -gn archivebox-smoke-user)"
+            sudo mkdir "$root_tool_dir/data-mixed-owner"
+            sudo chown archivebox-smoke-user:root "$root_tool_dir/data-mixed-owner"
+            sudo env PATH="$root_tool_dir/bin:$PATH" \
+              bash -c "cd '$root_tool_dir/data-mixed-owner' && archivebox init"
+            sudo stat -c %U:%G "$root_tool_dir/data-mixed-owner" | grep -qx "archivebox-smoke-user:$smoke_user_group"
+            if sudo find "$root_tool_dir/data-mixed-owner" -xdev -user root -print -quit | grep -q .; then
+              echo "Root-owned files remain in $root_tool_dir/data-mixed-owner" >&2
+              exit 1
+            fi
+            sudo rm -rf "$root_tool_dir"
+          fi

.github/workflows/release-runner.yml

@@ -1,49 +0,0 @@
-name: Release State
-
-on:
-  push:
-    branches:
-      - '**'
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  id-token: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  release-state:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: true
-          ref: ${{ github.ref_name }}
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - uses: astral-sh/setup-uv@v6
-        with:
-          enable-cache: false
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Configure git identity
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-
-      - name: Run release script
-        env:
-          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
-          GH_TOKEN: ${{ github.token }}
-          PYPI_PAT_SECRET: ${{ secrets.PYPI_PAT_SECRET }}
-        run: ./bin/release.sh

.github/workflows/release.yml

@@ -1,41 +1,324 @@
 name: Release
 
-# Orchestrates the full release pipeline:
-#   1. Build and publish pip package to PyPI
-#   2. Build Homebrew formula after pip
-#   3. Build Docker images in parallel
-#
-# Individual workflows also run on push for CI (see their own triggers).
-# Manual fallback for the hosted release pipeline. The normal prerelease loop
-# is run by bin/release.sh locally so published GitHub releases do not launch a
-# duplicate release job.
-
 on:
-  workflow_dispatch:
+  workflow_run:
+    workflows: [CI]
+    types: [completed]
+    branches: [dev, main]
 
-permissions:
-  contents: write
-  packages: write
-  id-token: write
+permissions: {}
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  group: release-${{ github.repository }}-${{ github.event.workflow_run.head_branch }}
+  cancel-in-progress: false
+
+env:
+  DOCKERHUB_IMAGE: archivebox/archivebox
+  GHCR_IMAGE: ghcr.io/archivebox/archivebox
+  RELEASE_BRANCH: ${{ github.event.workflow_run.head_branch }}
+  RELEASE_SHA: ${{ github.event.workflow_run.head_sha }}
+  CI_RUN_ID: ${{ github.event.workflow_run.id }}
 
 jobs:
-  pip:
-    name: Publish to PyPI
-    uses: ./.github/workflows/pip.yml
-    secrets: inherit
+  python-release:
+    name: Publish tested Python artifacts
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push'
+    runs-on: ubuntu-24.04
+    environment: pypi
+    permissions:
+      actions: read
+      contents: write
+      id-token: write
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+      tag: ${{ steps.version.outputs.tag }}
+      release_ready: ${{ steps.release-state.outputs.ready }}
+    steps:
+      - name: Checkout the tested commit
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 0
+          submodules: true
+          ref: ${{ env.RELEASE_SHA }}
+          token: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
 
-  homebrew:
-    name: Update Homebrew formula
-    needs: pip
-    uses: ./.github/workflows/homebrew.yml
-    secrets: inherit
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
 
-  docker:
-    name: Build Docker images
-    needs: pip
-    uses: ./.github/workflows/docker.yml
-    secrets: inherit
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+
+      - name: Resolve release binaries through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" >/dev/null
+          for name in uv gh git jq curl; do
+            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
+            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
+          done
+          {
+            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
+            echo "GH_BINARY=$ABXPKG_LIB_DIR/env/bin/gh"
+            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
+            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
+            echo "CURL_BINARY=$ABXPKG_LIB_DIR/env/bin/curl"
+          } >> "$GITHUB_ENV"
+
+      - name: Download exact tested distributions
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: python-distributions
+          path: ${{ runner.temp }}/python-distributions
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.CI_RUN_ID }}
+
+      - name: Record release version
+        id: version
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION="$($UV_BINARY run --no-cache --no-project python -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "tag=v$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Validate release branch and version
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION='${{ steps.version.outputs.version }}'
+          case "$RELEASE_BRANCH" in
+            main)
+              ;;
+            dev)
+              [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+rc[0-9]+$ ]] || {
+                echo "Refusing to publish ArchiveBox $VERSION from dev; only archivebox:main may publish non-prerelease versions." >&2
+                exit 1
+              }
+              ;;
+            *)
+              echo "Refusing to publish ArchiveBox $VERSION from unsupported branch $RELEASE_BRANCH; only archivebox:dev and archivebox:main may publish." >&2
+              exit 1
+              ;;
+          esac
+
+      - name: Publish the exact tested distributions
+        env:
+          GH_TOKEN: ${{ secrets.RELEASE_GH_TOKEN || github.token }}
+          RELEASE_DISTRIBUTIONS_DIR: ${{ runner.temp }}/python-distributions
+        run: ./bin/release.sh
+
+      - name: Confirm this tested commit owns the release tag
+        id: release-state
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          TARGET=""
+          while read -r sha _; do
+            TARGET="$sha"
+          done < <("$GIT_BINARY" ls-remote origin "refs/tags/${{ steps.version.outputs.tag }}")
+          if [[ "$TARGET" == "$RELEASE_SHA" ]]; then
+            echo "ready=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "ready=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Verify published PyPI package installs and runs
+        if: steps.release-state.outputs.ready == 'true'
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION='${{ steps.version.outputs.version }}'
+          TOOL_ROOT="$(mktemp -d)"
+          export UV_TOOL_DIR="$TOOL_ROOT/tools"
+          export UV_TOOL_BIN_DIR="$TOOL_ROOT/bin"
+          for attempt in {1..40}; do
+            if "$UV_BINARY" tool install --no-cache --force "archivebox==$VERSION"; then
+              break
+            fi
+            if [[ "$attempt" == 40 ]]; then
+              echo "archivebox==$VERSION did not become installable from PyPI within 10 minutes" >&2
+              exit 1
+            fi
+            sleep 15
+          done
+          INSTALLED_VERSION="$("$UV_TOOL_BIN_DIR/archivebox" --version)"
+          test "$INSTALLED_VERSION" = "$VERSION"
+
+  docker-release:
+    name: Publish tested multiarch image
+    if: >-
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'push' &&
+      needs.python-release.outputs.release_ready == 'true'
+    needs: python-release
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout the tested commit
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          fetch-depth: 1
+          ref: ${{ env.RELEASE_SHA }}
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.13"
+
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+
+      - name: Resolve release and Docker binaries through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:release_binaries" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries" >/dev/null
+          for name in uv git jq docker; do
+            test -L "$ABXPKG_LIB_DIR/env/bin/$name"
+            test -x "$ABXPKG_LIB_DIR/env/bin/$name"
+          done
+          {
+            echo "UV_BINARY=$ABXPKG_LIB_DIR/env/bin/uv"
+            echo "GIT_BINARY=$ABXPKG_LIB_DIR/env/bin/git"
+            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
+            echo "DOCKER_BINARY=$ABXPKG_LIB_DIR/env/bin/docker"
+          } >> "$GITHUB_ENV"
+
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+        with:
+          install: true
+
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      - name: Download exact tested image digests
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          path: /tmp/digests
+          pattern: digest-*
+          merge-multiple: true
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.CI_RUN_ID }}
+
+      - name: Verify digest set and collect tags
+        id: docker_meta
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          [[ "${#DIGESTS[@]}" -eq 2 ]]
+          for digest in "${DIGESTS[@]}"; do [[ "$digest" =~ ^[0-9a-f]{64}$ ]]; done
+
+          VERSION='${{ needs.python-release.outputs.version }}'
+          SHORT_SHA="${RELEASE_SHA::12}"
+          TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}^{}")"
+          TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
+          if [[ -z "$TAG_TARGET" ]]; then
+            TAG_TARGET="$($GIT_BINARY ls-remote origin "refs/tags/v${VERSION}")"
+            TAG_TARGET="${TAG_TARGET%%[[:space:]]*}"
+          fi
+          {
+            echo 'dockerhub_tags<<EOF'
+            if [[ "$RELEASE_BRANCH" == dev ]]; then
+              echo "${DOCKERHUB_IMAGE}:dev"
+            elif [[ "$RELEASE_BRANCH" == main ]]; then
+              echo "${DOCKERHUB_IMAGE}:latest"
+            fi
+            echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"
+            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${DOCKERHUB_IMAGE}:${VERSION}"
+            echo 'EOF'
+            echo 'ghcr_tags<<EOF'
+            if [[ "$RELEASE_BRANCH" == dev ]]; then
+              echo "${GHCR_IMAGE}:dev"
+            elif [[ "$RELEASE_BRANCH" == main ]]; then
+              echo "${GHCR_IMAGE}:latest"
+            fi
+            echo "${GHCR_IMAGE}:sha-${SHORT_SHA}"
+            [[ "$TAG_TARGET" == "$RELEASE_SHA" ]] && echo "${GHCR_IMAGE}:${VERSION}"
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Create Docker Hub manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          TAG_ARGS=()
+          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.dockerhub_tags }}'
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
+          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Create GHCR manifest
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file())))')
+          TAG_ARGS=()
+          while IFS= read -r tag; do [[ -n "$tag" ]] && TAG_ARGS+=(--tag "$tag"); done <<< '${{ steps.docker_meta.outputs.ghcr_tags }}'
+          REFS=()
+          for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
+          $DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
+
+      - name: Verify published Docker images run
+        shell: bash
+        run: |
+          set -Eeuo pipefail
+          VERSION='${{ needs.python-release.outputs.version }}'
+          for image in "${DOCKERHUB_IMAGE}:${VERSION}" "${GHCR_IMAGE}:${VERSION}"; do
+            "$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
+            INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
+            test "$INSTALLED_VERSION" = "$VERSION"
+          done
+
+      - name: Update Docker Hub README
+        uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: archivebox/archivebox
+
+  cascade:
+    name: Cascade verified release
+    needs:
+      - python-release
+      - docker-release
+    permissions:
+      contents: read
+    uses: ArchiveBox/monorepo/.github/workflows/cascade-release.yml@main
+    with:
+      upstream_package: archivebox
+      upstream_version: ${{ needs.python-release.outputs.version }}
+    secrets:
+      release_token: ${{ secrets.RELEASE_GH_TOKEN }}

.github/workflows/test-parallel.yml

@@ -1,311 +1,154 @@
 name: Parallel Tests
 
 on:
-  workflow_dispatch:
-  pull_request:
-    branches: [dev, main, master]
-  push:
-    branches: [dev]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  workflow_call:
 
 env:
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
+  CI_PYTHON_VERSION: "3.13"
 
 jobs:
   discover-tests:
-    name: Discover test files
+    name: Discover every test
     runs-on: ubuntu-24.04
     outputs:
-      test-files: ${{ steps.set-matrix.outputs.test-files }}
-      plugin-tests: ${{ steps.set-plugin-matrix.outputs.plugin-tests }}
+      tests: ${{ steps.discover.outputs.tests }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - name: Discover test files
-        id: set-matrix
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: ${{ env.CI_PYTHON_VERSION }}
+
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
+
+      - name: Resolve discovery tools through abxpkg
+        env:
+          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
         run: |
-          # Find all main test files
-          all_tests=$(find archivebox/tests -maxdepth 1 -name "test_*.py" -type f | sort)
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          test -n "$ABXPKG_VERSION"
+          mkdir -p "$ABXPKG_LIB_DIR/env/bin"
+          export PYTHON_BINARY="$pythonLocation/bin/python"
+          test -x "$PYTHON_BINARY"
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+            --install \
+            --lib="$ABXPKG_LIB_DIR" \
+            --deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:ci_binaries"
+          {
+            echo "PYTHON_BINARY=$ABXPKG_LIB_DIR/env/bin/python"
+            echo "JQ_BINARY=$ABXPKG_LIB_DIR/env/bin/jq"
+          } >> "$GITHUB_ENV"
+          test -L "$ABXPKG_LIB_DIR/env/bin/python"
+          test -x "$ABXPKG_LIB_DIR/env/bin/python"
+          test -L "$ABXPKG_LIB_DIR/env/bin/jq"
+          test -x "$ABXPKG_LIB_DIR/env/bin/jq"
 
-          # Create JSON array with test file info
-          json_array="["
-          first=true
-          for test_file in $all_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            # Extract a display name for the test
-            name="main/$(basename $test_file .py | sed 's/^test_//')"
-            needs_chromium=false
-            if grep -Eiq '(chrom|archivewebpage|PLUGINS=.*title|--plugins=.*title|SAVE_TITLE.*[Tt]rue)' "$test_file"; then
-              needs_chromium=true
-            fi
-
-            json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium}"
-          done
-          json_array+="]"
-
-          echo "test-files=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $all_tests | wc -w) test files"
-          echo "$json_array" | jq '.'
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Discover plugin tests
-        id: set-plugin-matrix
+      - name: Discover every test exactly once
+        id: discover
         run: |
-          plugin_tests=$(find abx-plugins/abx_plugins/plugins -maxdepth 2 -type d -name tests | sed 's#abx-plugins/abx_plugins/plugins/##; s#/tests##' | sort)
+          set -Eeuo pipefail
+          mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
+          test "${#output[@]}" -eq 2
+          echo "tests=${output[1]}" >> "$GITHUB_OUTPUT"
+          echo "${output[0]}"
+          "$JQ_BINARY" -e 'length > 0 and length <= 256 and ([.[].paths[]] | length == (unique | length))' <<< "${output[1]}"
 
-          json_array="["
-          first=true
-          for plugin_name in $plugin_tests; do
-            if [ "$first" = true ]; then
-              first=false
-            else
-              json_array+=","
-            fi
-
-            needs_chromium=false
-            if grep -Riq "chrom" "abx-plugins/abx_plugins/plugins/$plugin_name"; then
-              needs_chromium=true
-            fi
-
-            json_array+="{\"plugin\":\"$plugin_name\",\"name\":\"plugin/$plugin_name\",\"needs_chromium\":$needs_chromium}"
-          done
-          json_array+="]"
-
-          echo "plugin-tests=$json_array" >> $GITHUB_OUTPUT
-          echo "Found $(echo $plugin_tests | wc -w) plugin test suites"
-          echo "$json_array" | jq '.'
-
-  run-tests:
+  tests:
     name: ${{ matrix.test.name }}
-    runs-on: ubuntu-24.04
     needs: discover-tests
+    runs-on: ubuntu-24.04
     env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
+      ABXPKG_LIB_DIR: /tmp/archivebox-lib
       CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
+      DATA_DIR: /tmp/archivebox-test-data
       PERSONAS_DIR: /tmp/abx-personas
       CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
 
     strategy:
       fail-fast: false
       matrix:
-        test: ${{ fromJson(needs.discover-tests.outputs.test-files) }}
-        python: ["3.13"]
+        test: ${{ fromJson(needs.discover-tests.outputs.tests) }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v5
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
-          python-version: ${{ matrix.python }}
+          python-version: ${{ env.CI_PYTHON_VERSION }}
           architecture: x64
 
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
         with:
-          version: "latest"
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
 
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-
-      - name: Install Chrome through ArchiveBox
-        if: ${{ matrix.test.needs_chromium }}
+      - name: Resolve optional build dependencies through abxpkg
+        if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
         env:
-          MIN_CHROMIUM_MAJOR: "149"
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+          TEST_EXTRA: ${{ matrix.test.extra }}
+          TEST_PATHS: ${{ matrix.test.paths_arg }}
         run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-
-          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
-
-          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
-          if [ ! -x "$candidate" ]; then
-            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
-            exit 1
+          set -Eeuo pipefail
+          ABXPKG_VERSION="$(uv run --no-cache --no-project python -c 'import tomllib; print(next(package["version"] for package in tomllib.load(open("uv.lock", "rb"))["package"] if package["name"] == "abxpkg"))')"
+          test -n "$ABXPKG_VERSION"
+          dependency_sources=()
+          if [[ -n "$TEST_EXTRA" ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
           fi
-          version="$("$candidate" --version || true)"
-          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-          case "$major" in
-            ''|*[!0-9]*) major=0 ;;
-          esac
-          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
-            exit 1
+          if [[ "$TEST_PATHS" == *test_postgres_backend* ]]; then
+            dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:postgres_binaries")
           fi
-          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
-          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          uv run --no-cache --no-project --with "abxpkg==$ABXPKG_VERSION" abxpkg env \
+            --install \
+            --no-cache \
+            --json \
+            --lib="$ABXPKG_LIB_DIR" \
+            "${dependency_sources[@]}"
 
-      - name: Run test - ${{ matrix.test.name }}
-        run: |
-          mkdir -p tests/out
-          set +e
-          uv run --no-sync --no-sources pytest -xvs "${{ matrix.test.path }}" --basetemp=tests/out --ignore=archivebox/pkgs
-          status=$?
-          set -e
-          if [ "$status" -eq 5 ]; then
-            echo "No tests collected from ${{ matrix.test.path }}; treating as an empty test module."
-            exit 0
-          fi
-          exit "$status"
-
-  plugin-tests:
-    name: ${{ matrix.plugin.name }}
-    runs-on: ubuntu-24.04
-    needs: discover-tests
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
-      CHROME_HEADLESS: "true"
-      CHROME_SANDBOX: "false"
-      PERSONAS_DIR: /tmp/abx-personas
-      CHROME_USER_DATA_DIR: /tmp/abx-personas/Default/chrome_profile
-
-    strategy:
-      fail-fast: false
-      matrix:
-        plugin: ${{ fromJson(needs.discover-tests.outputs.plugin-tests) }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      - name: Set up Python 3.13
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-          architecture: x64
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      - name: Cache uv
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-3.13-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: git ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.1
-
-      - name: Install dependencies with uv
-        run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-
-      - name: Install Chrome through ArchiveBox
-        if: ${{ matrix.plugin.needs_chromium }}
+      - name: Install test dependencies
         env:
-          MIN_CHROMIUM_MAJOR: "149"
-          LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib
-          ABXPKG_INSTALL_TIMEOUT: "900"
-          ABX_CI_DATA_DIR: ${{ runner.temp }}/archivebox-ci-data
+          TEST_EXTRA: ${{ matrix.test.extra }}
         run: |
-          set -euo pipefail
-          mkdir -p "$LIB_DIR" "$ABX_CI_DATA_DIR"
-          echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV"
-          echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV"
-
-          uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox install chrome
-
-          candidate="$(uv run --project "$GITHUB_WORKSPACE" --directory "$ABX_CI_DATA_DIR" --no-sync --no-sources archivebox shell -c 'from archivebox.machine.models import Binary; binary = Binary.objects.filter(name="chromium", status="installed").order_by("-modified_at").first(); print(binary.abspath if binary else "")' | tail -n 1)"
-          if [ ! -x "$candidate" ]; then
-            echo "ArchiveBox did not install an executable Chromium binary: ${candidate:-not found}" >&2
-            exit 1
+          set -Eeuo pipefail
+          extra_args=()
+          if [[ -n "$TEST_EXTRA" ]]; then
+            extra_args=(--extra "$TEST_EXTRA")
           fi
-          version="$("$candidate" --version || true)"
-          major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)"
-          case "$major" in
-            ''|*[!0-9]*) major=0 ;;
-          esac
-          if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then
-            echo "ArchiveBox installed Chromium is too old: ${version:-unknown}" >&2
-            exit 1
-          fi
-          echo "Using ArchiveBox-installed Chromium: $candidate ($version)"
-          echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV"
+          uv sync --no-cache --locked --dev "${extra_args[@]}"
 
-      - name: Run plugin tests - ${{ matrix.plugin.name }}
+      - name: Run ${{ matrix.test.name }}
         env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
+          TEST_PATHS: ${{ matrix.test.paths_arg }}
         run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh "${{ matrix.plugin.plugin }}" --no-coverage
+          set -Eeuo pipefail
+          read -r -a test_paths <<< "$TEST_PATHS"
+          for test_path in "${test_paths[@]}"; do
+            test -f "$test_path"
+          done
+          for test_path in "${test_paths[@]}"; do
+            test_name="$(basename "$test_path" .py)"
+            test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
+            export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
+            export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
+            export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
+            mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
+            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
+              --basetemp="$RUNNER_TEMP/pytest/$test_slug"
+          done

.github/workflows/test.yml

@@ -1,156 +1,63 @@
-name: Run tests
-on: [push]
+name: Install and CLI Compatibility
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+on:
+  workflow_call:
 
 env:
   DOCKER_IMAGE: archivebox-ci
+  ARCHIVEBOX_IMAGE: archivebox-ci
   PYTHONIOENCODING: utf-8
   PYTHONLEGACYWINDOWSSTDIO: utf-8
   USE_COLOR: False
   UV_NO_SOURCES: "1"
 
 jobs:
-  python_tests:
+  install_cli_platform:
+    name: CLI install / ${{ matrix.os_name }} / Python ${{ matrix.python }}
     runs-on: ${{ matrix.os }}
-    env:
-      PYTHONPATH: ${{ github.workspace }}/abxpkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl
 
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
-        # os: [ubuntu-22.04, macos-latest, windows-latest]
-        python: ["3.13"]
+        include:
+          - os: ubuntu-24.04
+            os_name: linux
+            python: "3.14"
+          - os: macos-15
+            os_name: macOS
+            python: "3.13"
+          - os: macos-15
+            os_name: macOS
+            python: "3.14"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
           submodules: true
           fetch-depth: 1
 
-      - name: Clone abxpkg
-        run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git abxpkg
-
-      - name: Clone abx-plugins
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins
-
-      - name: Clone abx-dl
-        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl
-
-      ### Setup Python & JS Languages
+      ### Set up Python
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: ${{ matrix.python }}
-          architecture: x64
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v4
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
         with:
-          version: "latest"
-
-      - name: Set up Node JS
-        uses: actions/setup-node@v4
-        with:
-          node-version: 22
-
-      ### Install Python & JS Dependencies
-      - name: Cache uv
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/uv
-          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-uv-
-
-      - uses: awalsh128/cache-apt-pkgs-action@latest
-        with:
-          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
-          version: 1.0
+          version: "0.11.3"
+          enable-cache: false
+          cache-dependency-glob: uv.lock
 
       - name: Install dependencies with uv
+        run: uv sync --no-cache --locked --dev
+
+      - name: ArchiveBox full install check
         run: |
-          uv venv
-          uv pip install --group dev -e ./abxpkg -e ./abx-plugins -e ./abx-dl -e ".[all]"
-
-      ### Run the tests
-      - name: Directory listing for debugging
-        run: |
-          pwd
-          ls
-
-      - name: Archivebox version
-        run: |
-          mkdir -p tests/out/data
-          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version
-
-      - name: Test built package with pytest
-        # TODO: remove this exception for windows once we get tests passing on that platform
-        if: ${{ !contains(matrix.os, 'windows') }}
-        run: |
-          mkdir -p tests/out
-          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs
-
-      - name: Run plugin tests
-        if: ${{ !contains(matrix.os, 'windows') }}
-        env:
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
-          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
-        run: |
-          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage
-
-  docker_tests:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-          fetch-depth: 1
-
-      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
-      # - uses: satackey/action-docker-layer-caching@v0.0.8
-
-      - name: Build image
-        run: |
-          docker build . -t "$DOCKER_IMAGE"
-
-      - name: Init data dir
-        run: |
-          mkdir "${{ github.workspace }}/data"
-          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init
-
-      - name: Run test server
-        run: |
-          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
-          docker run --name www-nginx -p 80:80 -d nginx
-
-      - name: Add link
-        run: |
-          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local
-
-      - name: Add stdin link
-        run: |
-          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add
-
-      - name: List links
-        run: |
-          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
-          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
-
-      - name: Start docker-compose stack
-        run: |
-          docker-compose run archivebox init
-          docker-compose up -d
-          sleep 5
-          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
-          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'
-
-      - name: Check added urls show up in index
-        run: |
-          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
-          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
-          docker-compose down || true
+          set -Eeuo pipefail
+          DATA_DIR="$PWD/tests/out/data"
+          mkdir -p "$DATA_DIR"
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox init
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox install
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox version
+          uv run --no-cache --directory "$DATA_DIR" --no-sync --no-sources archivebox status

.github/workflows/update-homebrew-tap.yml

@@ -1,35 +0,0 @@
-name: Update Homebrew tap
-
-on:
-  push:
-    branches: [dev]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: update-homebrew-tap-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  dispatch:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Dispatch tap update
-        env:
-          HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
-        run: |
-          if [ -z "$HOMEBREW_TAP_TOKEN" ]; then
-            echo "HOMEBREW_TAP_TOKEN is not configured; tap hourly fallback will update the formula."
-            exit 0
-          fi
-
-          curl --fail-with-body \
-            -X POST \
-            -H "Accept: application/vnd.github+json" \
-            -H "Authorization: Bearer ${HOMEBREW_TAP_TOKEN}" \
-            -H "X-GitHub-Api-Version: 2022-11-28" \
-            https://api.github.com/repos/ArchiveBox/homebrew-archivebox/dispatches \
-            -d '{"event_type":"archivebox-dev-updated"}'

.gitignore

@@ -8,6 +8,8 @@ __pycache__/
 .mypy_cache/
 .eggs/
 tests/out/
+/docs/screenshots/
+/publicsite/screenshots/
 
 # Coverage
 .coverage

AGENTS.md

@@ -0,0 +1,80 @@
+# ArchiveBox Agent Guide
+
+ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev` branch.
+
+## Shared Standards
+
+- Use `uv` and `uv run` for Python commands. Do not use system `python`, direct `.venv/bin/python`, or `pip` commands.
+- Prefer existing repo patterns, helper APIs, fixtures, scripts, and command surfaces.
+- Keep edits focused and minimal. Do not add wrappers, shims, aliases, or extra abstraction layers unless the current code path requires them.
+- Do not weaken assertions, skip tests, xfail tests, or accept flaky behavior.
+- No mocks, monkeypatches, fakes, simulated handlers, fake binaries, fake hooks, fake buses, or direct shortcuts around user-facing flows.
+- Tests and verification should use real CLI commands, REST/API calls, browser UI flows, real hooks, real installs, real subprocesses, real DB rows, real files, and existing fixtures.
+- Assertions must verify real correctness: exit codes, returned values, DB state, filesystem contents, field values, rendered output, and side effects.
+- Start behavior fixes with a red failing test when a test is requested or practical.
+- Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
+- Read `README.md` for the full setup, CLI, Docker, API, and release surface.
+
+## Development Setup
+
+```bash
+uv sync --dev --all-extras
+mkdir -p data
+cd data
+uv run --project .. archivebox init --install
+```
+
+Run collection commands from inside an initialized data directory:
+
+```bash
+cd data
+uv run --project .. archivebox status
+uv run --project .. archivebox add --plugins=parse_txt_urls 'https://example.com/'
+uv run --project .. archivebox run
+```
+
+## User-Facing Setup
+
+Recommended CLI install:
+
+```bash
+uv tool install --python 3.13 --prerelease explicit --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
+mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init --install
+archivebox add --plugins=parse_txt_urls 'https://example.com/'
+```
+
+Alternative install methods:
+
+- Docker Compose / Docker
+- Homebrew
+- Debian package
+- pip
+
+## Basic Usage
+
+```bash
+cd ~/archivebox/data
+archivebox version
+archivebox help
+archivebox status
+archivebox install
+archivebox add --plugins=parse_txt_urls 'https://example.com/docs-basic-usage'
+archivebox list --json --with-headers
+archivebox search 'example'
+archivebox update --filter-type=domain example.com
+archivebox remove --yes --delete --filter-type=exact 'https://example.com/docs-basic-usage'
+archivebox run
+```
+
+## Verification
+
+Use targeted tests for focused work:
+
+```bash
+uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
+uv run prek run --all-files
+```
+
+Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.

CLAUDE.md

@@ -1,497 +0,0 @@
-# Claude Code Development Guide for ArchiveBox
-
-## Quick Start
-
-```bash
-# Set up dev environment (always use uv, never pip directly)
-uv sync --dev --all-extras
-
-# Run tests as non-root user (required - ArchiveBox always refuses to run as root)
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/ -v'
-```
-
-## Development Environment Setup
-
-### Prerequisites
-- Python 3.11+ (3.13 recommended)
-- uv package manager
-- A non-root user for running tests (e.g., `testuser`)
-
-### Install Dependencies
-```bash
-uv sync --dev --all-extras  # Always use uv, never pip directly
-```
-
-### Activate Virtual Environment
-```bash
-source .venv/bin/activate
-```
-
-### Common Gotchas
-
-#### File Permissions
-New files created by root need permissions fixed for testuser:
-```bash
-chmod 644 archivebox/tests/test_*.py
-```
-
-#### DATA_DIR Environment Variable
-ArchiveBox commands must run inside a data directory. Tests use temp directories - the `run_archivebox()` helper sets `DATA_DIR` automatically.
-
-## Code Style Guidelines
-
-### Naming Conventions for Grep-ability
-Use consistent naming for everything to enable easy grep-ability and logical grouping:
-
-**Principle**: Fewest unique names. If you must create a new unique name, make it grep and group well.
-
-**Examples**:
-```python
-# Filesystem migration methods - all start with fs_
-def fs_migration_needed() -> bool: ...
-def fs_migrate() -> None: ...
-def _fs_migrate_from_0_7_0_to_0_8_0() -> None: ...
-def _fs_migrate_from_0_8_0_to_0_9_0() -> None: ...
-def _fs_next_version(current: str) -> str: ...
-
-# Logging methods - ALL must start with log_ or _log
-def log_migration_start(snapshot_id: str) -> None: ...
-def _log_error(message: str) -> None: ...
-def log_validation_result(ok: bool, msg: str) -> None: ...
-```
-
-**Rules**:
-- Group related functions with common prefixes
-- Use `_` prefix for internal/private helpers within the same family
-- ALL logging-related methods MUST start with `log_` or `_log`
-- Search for all migration functions: `grep -r "def.*fs_.*(" archivebox/`
-- Search for all logging: `grep -r "def.*log_.*(" archivebox/`
-
-### Minimize Unique Names and Data Structures
-**Do not invent new data structures, variable names, or keys if possible.** Try to use existing field names and data structures exactly to keep the total unique data structures and names in the codebase to an absolute minimum.
-
-**Example - GOOD**:
-```python
-# Binary has overrides field
-binary = Binary(overrides={'TIMEOUT': '60s'})
-
-# Binary reuses the same field name and structure
-class Binary(models.Model):
-    overrides = models.JSONField(default=dict)  # Same name, same structure
-```
-
-**Example - BAD**:
-```python
-# Don't invent new names like custom_bin_cmds, binary_overrides, etc.
-class Binary(models.Model):
-    custom_bin_cmds = models.JSONField(default=dict)  # ❌ New unique name
-```
-
-**Principle**: If you're storing the same conceptual data (e.g., `overrides`), use the same field name across all models and keep the internal structure identical. This makes the codebase predictable and reduces cognitive load.
-
-## Testing
-
-### CRITICAL: Never Run as Root
-ArchiveBox has a root check that prevents running as root user. All ArchiveBox commands (including tests) must run as non-root user inside a data directory:
-
-```bash
-# Run all migration tests
-sudo -u testuser bash -c 'source /path/to/.venv/bin/activate && python -m pytest archivebox/tests/test_migrations_*.py -v'
-
-# Run specific test file
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -v'
-
-# Run single test
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_fresh.py::TestFreshInstall::test_init_creates_database -xvs'
-```
-
-### Test File Structure
-```
-archivebox/tests/
-├── test_migrations_helpers.py    # Schemas, seeding functions, verification helpers
-├── test_migrations_fresh.py      # Fresh install tests
-├── test_migrations_04_to_09.py   # 0.4.x → 0.9.x migration tests
-├── test_migrations_07_to_09.py   # 0.7.x → 0.9.x migration tests
-└── test_migrations_08_to_09.py   # 0.8.x → 0.9.x migration tests
-```
-
-### Test Writing Standards
-
-#### NO MOCKS - Real Tests Only
-Tests must exercise real code paths:
-- Create real SQLite databases with version-specific schemas
-- Seed with realistic test data
-- Run actual `python -m archivebox` commands via subprocess
-- Query SQLite directly to verify results
-
-**If something is hard to test**: Modify the implementation to make it easier to test, or fix the underlying issue. Never mock, skip, simulate, or exit early from a test because you can't get something working inside the test.
-
-#### NO SKIPS
-Never use `@skip`, `skipTest`, or `pytest.mark.skip`. Every test must run. If a test is difficult, fix the code or test environment - don't disable the test.
-
-#### Strict Assertions
-- `init` command must return exit code 0 (not `[0, 1]`)
-- Verify ALL data is preserved, not just "at least one"
-- Use exact counts (`==`) not loose bounds (`>=`)
-
-### Example Test Pattern
-```python
-def test_migration_preserves_snapshots(self):
-    """Migration should preserve all snapshots."""
-    result = run_archivebox(self.work_dir, ['init'], timeout=45)
-    self.assertEqual(result.returncode, 0, f"Init failed: {result.stderr}")
-
-    ok, msg = verify_snapshot_count(self.db_path, expected_count)
-    self.assertTrue(ok, msg)
-```
-
-### Testing Gotchas
-
-#### Extractors Disabled for Speed
-Tests disable all extractors via environment variables for faster execution:
-```python
-env['SAVE_TITLE'] = 'False'
-env['SAVE_FAVICON'] = 'False'
-# ... etc
-```
-
-#### Timeout Settings
-Use appropriate timeouts for migration tests (45s for init, 60s default).
-
-### Plugin Testing & Code Coverage
-
-**Target: 80-90% coverage** for critical plugins (screenshot, chrome, singlefile, dom)
-
-```bash
-# Run plugin tests with coverage (both Python + JavaScript)
-bash bin/test_plugins.sh screenshot
-
-# View coverage reports
-bash bin/test_plugins.sh --coverage-report
-# Or individual reports:
-coverage report --show-missing --include='archivebox/plugins/*' --omit='*/tests/*'
-```
-
-#### Plugin Test Structure
-
-Tests are **completely isolated** from ArchiveBox - they replicate production directory structure in temp dirs:
-
-```python
-# Correct production paths:
-# Crawl:    DATA_DIR/users/{username}/crawls/YYYYMMDD/example.com/{crawl-id}/{plugin}/
-# Snapshot: DATA_DIR/users/{username}/snapshots/YYYYMMDD/example.com/{snapshot-uuid}/{plugin}/
-
-with tempfile.TemporaryDirectory() as tmpdir:
-    data_dir = Path(tmpdir)
-
-    # Crawl-level plugin (e.g., chrome launcher)
-    crawl_dir = data_dir / 'users' / 'testuser' / 'crawls' / '20240101' / 'example.com' / 'crawl-123'
-    chrome_dir = crawl_dir / 'chrome'
-    chrome_dir.mkdir(parents=True)
-
-    # Snapshot-level plugin (e.g., screenshot)
-    snapshot_dir = data_dir / 'users' / 'testuser' / 'snapshots' / '20240101' / 'example.com' / 'snap-456'
-    screenshot_dir = snapshot_dir / 'screenshot'
-    screenshot_dir.mkdir(parents=True)
-
-    # Run hook in its output directory
-    result = subprocess.run(
-        ['node', str(SCREENSHOT_HOOK), '--url=https://example.com'],
-        cwd=str(screenshot_dir),
-        env={**get_test_env(), 'EXTRA_CONTEXT': '{"snapshot_id":"snap-456"}'},
-        capture_output=True,
-        timeout=120
-    )
-```
-
-#### Coverage Improvement Loop
-
-To improve from ~20% to 80%+:
-
-1. **Run tests**: `bash bin/test_plugins.sh screenshot` → Shows: `19.1% (13/68 ranges)`
-2. **Identify gaps**: Check hook file for untested paths (session connection vs fallback, config branches, error cases)
-3. **Add tests**: Test both execution paths (connect to session + launch own browser), skip conditions, error cases, config variations
-4. **Verify**: Re-run tests → Should show: `85%+ (58+/68 ranges)`
-
-**Critical**: JavaScript hooks have TWO paths that both must be tested (connect to session ~50% + launch browser ~30% + shared ~20%). Testing only one path = max 50% coverage possible!
-
-## Database Migrations
-
-### Generate and Apply Migrations
-```bash
-# Generate migrations (run from archivebox subdirectory)
-cd archivebox
-./manage.py makemigrations
-
-# Apply migrations to test database
-cd data/
-archivebox init
-```
-
-### Schema Versions
-- **0.4.x**: First Django version. Tags as comma-separated string, no ArchiveResult model
-- **0.7.x**: Tag model with M2M, ArchiveResult model, AutoField PKs
-- **0.8.x**: Crawl/Seed models, UUID PKs, status fields, depth/retry_at
-- **0.9.x**: Seed model removed, seed_id FK removed from Crawl
-
-### Testing a Migration Path
-1. Create SQLite DB with source version schema (from `test_migrations_helpers.py`)
-2. Seed with realistic test data using `seed_0_X_data()`
-3. Run `archivebox init` to trigger migrations
-4. Verify data preservation with `verify_*` functions
-5. Test CLI commands work post-migration (`status`, `list`, `add`, etc.)
-
-### Squashed Migrations
-When testing 0.8.x (dev branch), you must record ALL replaced migrations:
-```python
-# The squashed migration replaces these - all must be recorded
-('core', '0023_alter_archiveresult_options_archiveresult_abid_and_more'),
-('core', '0024_auto_20240513_1143'),
-# ... all 52 migrations from 0023-0074 ...
-('core', '0023_new_schema'),  # Also record the squashed migration itself
-```
-
-### Migration Strategy
-- Squashed migrations for clean installs
-- Individual migrations recorded for upgrades from dev branch
-- `replaces` attribute in squashed migrations lists what they replace
-
-### Migration Gotchas
-
-#### Circular FK References in Schemas
-SQLite handles circular references with `IF NOT EXISTS`. Order matters less than in other DBs.
-
-## Plugin System Architecture
-
-### Plugin Dependency Rules
-
-Like other plugins, chrome plugins **ARE NOT ALLOWED TO DEPEND ON ARCHIVEBOX OR DJANGO**.
-However, they are allowed to depend on two shared files ONLY:
-- `archivebox/plugins/chrome/chrome_utils.js` ← source of truth API for all basic chrome ops
-- `archivebox/plugins/chrome/tests/chrome_test_utils.py` ← use for your tests, do not implement launching/killing/pid files/cdp/etc. in python, just extend this file as needed.
-
-### Chrome-Dependent Plugins
-
-Many plugins depend on Chrome/Chromium via CDP (Chrome DevTools Protocol). When checking for script name references or debugging Chrome-related issues, check these plugins:
-
-**Main puppeteer-based chrome installer + launcher plugin**:
-- `chrome` - Core Chrome integration (CDP, launch, navigation)
-
-**Metadata extraction using chrome/chrome_utils.js / CDP**:
-- `dns` - DNS resolution info
-- `ssl` - SSL certificate info
-- `headers` - HTTP response headers
-- `redirects` - Capture redirect chains
-- `staticfile` - Direct file downloads (e.g. if the url itself is a .png, .exe, .zip, etc.)
-- `responses` - Capture network responses
-- `consolelog` - Capture console.log output
-- `title` - Extract page title
-- `accessibility` - Extract accessibility tree
-- `seo` - Extract SEO metadata
-
-**Extensions installed using chrome/chrome_utils.js / controlled using CDP**:
-- `ublock` - uBlock Origin ad blocking
-- `istilldontcareaboutcookies` - Cookie banner dismissal
-- `twocaptcha` - 2captcha CAPTCHA solver integration
-
-**Page-alteration plugins to prepare the content for archiving**:
-- `modalcloser` - Modal dialog dismissal
-- `infiniscroll` - Infinite scroll handler
-
-**Main Extractor Outputs**:
-- `dom` - DOM snapshot extraction
-- `pdf` - Generate PDF snapshots
-- `screenshot` - Generate screenshots
-- `singlefile` - SingleFile archival, can be single-file-cli that launches chrome, or singlefile extension running inside chrome
-
-**Crawl URL parsers** (post-process dom.html, singlefile.html, staticfile, responses, headers, etc. for URLs to re-emit as new queued Snapshots during recursive crawling):
-- `parse_dom_outlinks` - Extract outlinks from DOM (special, uses CDP to directly query browser)
-- `parse_html_urls` - Parse URLs from HTML (doesn't use chrome directly, just reads dom.html)
-- `parse_jsonl_urls` - Parse URLs from JSONL (doesn't use chrome directly, just reads dom.html)
-- `parse_netscape_urls` - Parse Netscape bookmark format (doesn't use chrome directly, just reads dom.html)
-
-### Finding Chrome-Dependent Plugins
-
-```bash
-# Find all files containing "chrom" (case-insensitive)
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d: -f1 | sort -u
-
-# Or get just the plugin names
-grep -ri "chrom" archivebox/plugins/*/on_*.* --include="*.*" 2>/dev/null | cut -d/ -f3 | sort -u
-```
-
-**Note**: This list may not be complete. Always run the grep command above when checking for Chrome-related script references or debugging Chrome integration issues.
-
-## Architecture Notes
-
-### Crawl Model (0.9.x)
-- Crawl groups multiple Snapshots from a single `add` command
-- Each `add` creates one Crawl with one or more Snapshots
-- Seed model was removed - crawls now store URLs directly
-
-## Code Coverage
-
-### Overview
-
-Coverage tracking is enabled for passive collection across all contexts:
-- Unit tests (pytest)
-- Integration tests
-- Dev server (manual testing)
-- CLI usage
-
-Coverage data accumulates in `.coverage` file and can be viewed/analyzed to find dead code.
-
-### Install Coverage Tools
-
-```bash
-uv sync --dev  # Installs pytest-cov and coverage
-```
-
-### Running with Coverage
-
-#### Unit Tests
-```bash
-# Run tests with coverage
-pytest --cov=archivebox --cov-report=term archivebox/tests/
-
-# Or run specific test file
-pytest --cov=archivebox --cov-report=term archivebox/tests/test_migrations_08_to_09.py
-```
-
-#### Dev Server with Coverage
-```bash
-# Start dev server with coverage tracking
-coverage run --parallel-mode -m archivebox server
-
-# Or CLI commands
-coverage run --parallel-mode -m archivebox init
-coverage run --parallel-mode -m archivebox add https://example.com
-```
-
-#### Manual Testing (Always-On)
-To enable coverage during ALL Python executions (passive tracking):
-
-```bash
-# Option 1: Use coverage run wrapper
-coverage run --parallel-mode -m archivebox [command]
-
-# Option 2: Set environment variable (tracks everything)
-export COVERAGE_PROCESS_START=pyproject.toml
-# Now all Python processes will track coverage
-archivebox server
-archivebox add https://example.com
-```
-
-### Viewing Coverage
-
-#### Text Report (Quick View)
-```bash
-# Combine all parallel coverage data
-coverage combine
-
-# View summary
-coverage report
-
-# View detailed report with missing lines
-coverage report --show-missing
-
-# View specific file
-coverage report --include="archivebox/core/models.py" --show-missing
-```
-
-#### JSON Report (LLM-Friendly)
-```bash
-# Generate JSON report
-coverage json
-
-# View the JSON
-cat coverage.json | jq '.files | keys'  # List all files
-
-# Find files with low coverage
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 50) | "\(.key): \(.value.summary.percent_covered)%"'
-
-# Find completely uncovered files (dead code candidates)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# Get missing lines for a specific file
-cat coverage.json | jq '.files["archivebox/core/models.py"].missing_lines'
-```
-
-#### HTML Report (Visual)
-```bash
-# Generate interactive HTML report
-coverage html
-
-# Open in browser
-open htmlcov/index.html
-```
-
-### Isolated Runs
-
-To measure coverage for specific scenarios:
-
-```bash
-# 1. Reset coverage data
-coverage erase
-
-# 2. Run your isolated test/scenario
-pytest --cov=archivebox archivebox/tests/test_migrations_fresh.py
-# OR
-coverage run --parallel-mode -m archivebox add https://example.com
-
-# 3. View results
-coverage combine
-coverage report --show-missing
-
-# 4. Optionally export for analysis
-coverage json
-```
-
-### Finding Dead Code
-
-```bash
-# 1. Run comprehensive tests + manual testing to build coverage
-pytest --cov=archivebox archivebox/tests/
-coverage run --parallel-mode -m archivebox server  # Use the app manually
-coverage combine
-
-# 2. Find files with 0% coverage (strong dead code candidates)
-coverage json
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered == 0) | .key'
-
-# 3. Find files with <10% coverage (likely dead code)
-cat coverage.json | jq -r '.files | to_entries[] | select(.value.summary.percent_covered < 10) | "\(.key): \(.value.summary.percent_covered)%"' | sort -t: -k2 -n
-
-# 4. Generate detailed report for analysis
-coverage report --show-missing > coverage_report.txt
-```
-
-### Tips
-
-- **Parallel mode** (`--parallel-mode`): Allows multiple processes to track coverage simultaneously without conflicts
-- **Combine**: Always run `coverage combine` before viewing reports to merge parallel data
-- **Reset**: Use `coverage erase` to start fresh for isolated measurements
-- **Branch coverage**: Enabled by default - tracks if both branches of if/else are executed
-- **Exclude patterns**: Config in `pyproject.toml` excludes tests, migrations, type stubs
-
-## Debugging Tips
-
-### Check Migration State
-```bash
-sqlite3 /path/to/index.sqlite3 "SELECT app, name FROM django_migrations WHERE app='core' ORDER BY id;"
-```
-
-### Check Table Schema
-```bash
-sqlite3 /path/to/index.sqlite3 "PRAGMA table_info(core_snapshot);"
-```
-
-### Verbose Test Output
-```bash
-sudo -u testuser bash -c 'source .venv/bin/activate && python -m pytest archivebox/tests/test_migrations_08_to_09.py -xvs 2>&1 | head -200'
-```
-
-### Kill Zombie Chrome Processes
-```bash
-./bin/kill_chrome.sh
-```

Dockerfile

@@ -2,7 +2,7 @@
 
 # Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
 # abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
+# ArchiveBox owns sonic, supervisor, Django, and the app runtime.
 # Build abx-dl first, then point this file at it:
 #   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
 #       --build-context abxbus=../abxbus \
@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.11.272 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.11.272
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
@@ -22,6 +22,7 @@ ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
+ARG ARCHIVEBOX_COMMIT_HASH=""
 
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
@@ -41,17 +42,16 @@ ENV PYTHON_VERSION=3.13 \
     NODE_VERSION=24
 
 ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
+    DEFAULT_ARCHIVEBOX_UID=911 \
+    DEFAULT_ARCHIVEBOX_GID=911 \
     IN_DOCKER=True
 
 ENV CODE_DIR=/app \
     DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
+    CONFIG_DIR=/opt/archivebox \
     ABXPKG_LIB_DIR=/opt/archivebox/lib \
     PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
     PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
     CHROME_HEADLESS=true \
     CHROME_SANDBOX=false \
     CHROME_ISOLATION=crawl \
@@ -65,20 +65,16 @@ ENV TMP_DIR=/tmp/archivebox \
 
 ENV HOME=/home/archivebox \
     XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/home/archivebox/.cache \
-    ABXPKG_INSTALL_TIMEOUT=600 \
-    ABXPKG_POSTINSTALL_SCRIPTS=True \
-    ABXPKG_MIN_RELEASE_AGE=0 \
+    XDG_CACHE_HOME=/opt/archivebox/lib/cache \
     TIMEOUT=600
 
 ENV UV_COMPILE_BYTECODE=false \
     UV_PYTHON_PREFERENCE=managed \
     UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \
     UV_LINK_MODE=copy \
     UV_PROJECT_ENVIRONMENT=/venv \
     VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
+    PATH="/venv/bin:/opt/node/bin:$PATH"
 
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
 WORKDIR "$CODE_DIR"
@@ -89,50 +85,97 @@ RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
     && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
     && uname -a \
     && sed -n '1,7p' /etc/os-release \
-    && which node && node --version \
-    && which uv && uv self version \
+    && abxpkg load --binproviders=env node \
+    && abxpkg load --binproviders=env uv \
     ) | tee -a /VERSION.txt
 
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends ripgrep \
-    && rm -rf /var/lib/apt/lists/*
-
 FROM archivebox-runtime-base AS archivebox-builder
 
 WORKDIR "$CODE_DIR"
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
     --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \
-    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
-    && /usr/bin/uv pip install setuptools pip wheel \
-    && /usr/bin/uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
+    <<'EOF'
+echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..."
+echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache
+echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends
+echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests
+rm -f /etc/apt/apt.conf.d/docker-clean
+apt-get update -qq
+apt-get install -qq -y --no-install-recommends \
+    build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+/usr/bin/uv venv --no-cache --clear /venv --python "${PYTHON_VERSION}"
+/usr/bin/uv pip install --no-cache setuptools pip wheel
+
+mkdir -p /tmp/archivebox-uv-project
+/venv/bin/python - <<'PY'
+from pathlib import Path
+import json
+import re
+import urllib.request
+
+source = Path("/app/pyproject.toml")
+target = Path("/tmp/archivebox-uv-project/pyproject.toml")
+text = source.read_text()
+text = text.replace(
+    'environments = ["sys_platform == \'darwin\'", "sys_platform == \'linux\'"]',
+    'environments = ["sys_platform == \'linux\'"]',
+)
+
+# Docker builds need the just-published internal abx wheels immediately, but
+# PyPI simple can lag the version JSON endpoints by tens of minutes. Generate a
+# Docker-only dependency view from the version JSON so the published package
+# metadata stays normal while image builds remain resumable after a release.
+for package in ("abxbus", "abxpkg", "abx-plugins", "abx-dl"):
+    match = re.search(
+        rf'"{re.escape(package)}(?P<operator>==|>=)(?P<version>[^"]+)"',
+        text,
+    )
+    if not match:
+        continue
+    version = match.group("version")
+    with urllib.request.urlopen(f"https://pypi.org/pypi/{package}/{version}/json", timeout=20) as response:
+        data = json.load(response)
+    wheel_url = next(url["url"] for url in data["urls"] if url["filename"].endswith(".whl"))
+    text = re.sub(
+        rf'"{re.escape(package)}(?:==|>=)[^"]+"',
+        f'"{package} @ {wheel_url}"',
+        text,
+        count=1,
+    )
+
+target.write_text(text)
+PY
+
+/usr/bin/uv sync \
+    --project /tmp/archivebox-uv-project \
+    --no-cache \
+    --no-dev \
+    --inexact \
+    --no-install-project \
+    --no-install-workspace \
+    --no-sources
+ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$ABXPKG_LIB_DIR" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
+"$ABXPKG_LIB_DIR/env/bin/find" /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
+while IFS= read -r -d '' native_library; do
+    magic=''
+    if IFS= read -r -N 4 magic < "$native_library" && [[ "$magic" == $'\x7fELF' ]]; then
+        "$ABXPKG_LIB_DIR/env/bin/strip" --strip-unneeded "$native_library" || exit $?
+    fi
+done < /tmp/archivebox-native-libraries
+rm -f /tmp/archivebox-native-libraries
+rm -f /venv/bin/uv /venv/bin/uvx
+abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get purge -y binutils build-essential gcc libldap2-dev libsasl2-dev libssl-dev
+abxpkg run --binproviders=env --lib="$ABXPKG_LIB_DIR" apt-get autoremove -y
+"$ABXPKG_LIB_DIR/env/bin/find" "$ABXPKG_LIB_DIR/env/bin" -maxdepth 1 -type l -name strip -delete
+rm -rf /var/lib/apt/lists/*
+EOF
 
 COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
+RUN echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
     && COMMIT_HASH="$( \
-        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
+        if [[ "$ARCHIVEBOX_COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then \
+            echo "$ARCHIVEBOX_COMMIT_HASH"; \
+        elif [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
             HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
             if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
                 echo "$HEAD_REF"; \
@@ -142,9 +185,10 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$T
             fi; \
         fi)" \
     && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
-    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && rm -rf "$CODE_DIR/.git"
+    && /usr/bin/uv pip install --no-cache --no-deps "$CODE_DIR" \
+    && rm -f /venv/bin/uv /venv/bin/uvx \
+    && find /venv -exec touch -h -d "@$(date +%s)" {} + \
+    && /usr/bin/uv pip show archivebox | tee -a /VERSION.txt
 
 FROM archivebox-runtime-base
 
@@ -162,51 +206,52 @@ LABEL name="archivebox" \
 COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
 COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
 
-COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python
 COPY --from=archivebox-builder /venv /venv
 COPY --from=archivebox-builder /app /app
 COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
 
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
+RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_ARCHIVEBOX_UID}..." \
+    && printf 'export PATH="/venv/bin:/opt/node/bin:$PATH"\n' > /etc/profile.d/archivebox-path.sh \
     && ln -sf /venv/bin/archivebox /usr/local/bin/archivebox \
+    && ln -sf /venv/bin/daphne /usr/local/bin/daphne \
+    && ln -sf /venv/bin/supervisord /usr/local/bin/supervisord \
+    && ln -sf /venv/bin/supervisorctl /usr/local/bin/supervisorctl \
     && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
     && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
     && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_UID" ]] || usermod -u "$DEFAULT_ARCHIVEBOX_UID" "$ARCHIVEBOX_USER" \
+    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_ARCHIVEBOX_GID" ]] || groupmod -g "$DEFAULT_ARCHIVEBOX_GID" "$ARCHIVEBOX_USER" \
+    && abxpkg load --binproviders=env sonic | tee -a /VERSION.txt \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && install -d -o "$DEFAULT_ARCHIVEBOX_UID" -g "$DEFAULT_ARCHIVEBOX_GID" "/home/$ARCHIVEBOX_USER" \
+    && chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$DATA_DIR" "$TMP_DIR" \
+    && chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" "$ABXPKG_LIB_DIR" \
     && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
+    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER ARCHIVEBOX_UID=$(id -u "$ARCHIVEBOX_USER") ARCHIVEBOX_GID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
+    && echo -e "TMP_DIR=$TMP_DIR\nABXPKG_LIB_DIR=$ABXPKG_LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
 
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
     && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
+    && HOME="$TMP_DIR" archivebox init \
+    && (chown "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" \
         "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
         "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
         "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
         "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+        "$CONFIG_DIR" "$CONFIG_DIR"/config.env "$CONFIG_DIR"/derived.env \
+        "$TMP_DIR" "$ABXPKG_LIB_DIR" "$XDG_CACHE_HOME" "$PLAYWRIGHT_BROWSERS_PATH" \
         2>/dev/null || true) \
     && find "$TMP_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
 
 RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
+    && chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
 
-RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
-    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
+RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
+    && stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
+    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox install \
     && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
     && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
 

Dockerfile.multistage

@@ -1,224 +0,0 @@
-# syntax=docker/dockerfile:1.7
-
-# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
-# abx-dl owns Python, Node, Chromium, and downloader plugin runtimes.
-# ArchiveBox owns ripgrep, sonic, supervisor, Django, and the app runtime.
-# Build abx-dl first, then point this file at it:
-#   docker buildx build ../abx-dl -f ../abx-dl/Dockerfile \
-#       --build-context abxbus=../abxbus \
-#       --build-context abxpkg=../abxpkg \
-#       --build-context abx-plugins=../abx-plugins \
-#       -t archivebox/abx-dl:dev
-#   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:latest \
-#       -t archivebox:multistage
-
-ARG ABX_DL_IMAGE=archivebox/abx-dl:latest
-
-FROM archivebox/sonic:1.4.9 AS sonic
-FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base
-
-ARG TARGETPLATFORM
-ARG TARGETOS
-ARG TARGETARCH
-ARG TARGETVARIANT
-
-ENV TZ=UTC \
-    LANGUAGE=en_US:en \
-    LC_ALL=C.UTF-8 \
-    LANG=C.UTF-8 \
-    DEBIAN_FRONTEND=noninteractive \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
-    PYTHONIOENCODING=UTF-8 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    PIP_NO_COMPILE=1 \
-    PIP_ONLY_BINARY=aiohttp \
-    npm_config_loglevel=error
-
-ENV PYTHON_VERSION=3.13 \
-    NODE_VERSION=22.22.3
-
-ENV ARCHIVEBOX_USER=archivebox \
-    DEFAULT_PUID=911 \
-    DEFAULT_PGID=911 \
-    IN_DOCKER=True
-
-ENV CODE_DIR=/app \
-    DATA_DIR=/data \
-    LIB_DIR=/opt/archivebox/lib \
-    ABXPKG_LIB_DIR=/opt/archivebox/lib \
-    PLAYWRIGHT_BROWSERS_PATH=/opt/archivebox/lib/playwright/cache \
-    PERSONAS_DIR=/data/personas \
-    CHROME_USER_DATA_DIR=/data/personas/Default/chrome_profile \
-    CHROME_HEADLESS=true \
-    CHROME_SANDBOX=false \
-    CHROME_ISOLATION=crawl \
-    CHROME_ARGS_EXTRA='["--disable-gpu","--disable-features=Translate,OptimizationGuideModelDownloading,MediaRouter"]'
-
-ENV TMP_DIR=/tmp/archivebox \
-    PIP_VENV_PYTHON=/venv/bin/python3 \
-    GOOGLE_API_KEY=no \
-    GOOGLE_DEFAULT_CLIENT_ID=no \
-    GOOGLE_DEFAULT_CLIENT_SECRET=no
-
-ENV HOME=/home/archivebox \
-    XDG_CONFIG_HOME=/home/archivebox/.config \
-    XDG_CACHE_HOME=/home/archivebox/.cache \
-    ABXPKG_INSTALL_TIMEOUT=600 \
-    ABXPKG_POSTINSTALL_SCRIPTS=True \
-    ABXPKG_MIN_RELEASE_AGE=0 \
-    TIMEOUT=600
-
-ENV UV_COMPILE_BYTECODE=false \
-    UV_PYTHON_PREFERENCE=managed \
-    UV_PYTHON_INSTALL_DIR=/opt/uv/python \
-    ARCHIVEBOX_PYTHON_INSTALL_DIR=/opt/archivebox/python \
-    UV_LINK_MODE=copy \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    VIRTUAL_ENV=/venv \
-    PATH="/venv/bin:/opt/node/bin:/opt/archivebox/lib/bin:$PATH"
-
-SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
-WORKDIR "$CODE_DIR"
-
-RUN cp /VERSION.txt /ABX-DL-VERSION.txt \
-    && (echo "[i] Docker build for ArchiveBox multistage starting..." \
-    && echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
-    && uname -a \
-    && sed -n '1,7p' /etc/os-release \
-    && which node && node --version \
-    && which uv && uv self version \
-    ) | tee -a /VERSION.txt
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    echo "[+] APT Installing ArchiveBox search dependency ripgrep for $TARGETPLATFORM..." \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends ripgrep \
-    && rm -rf /var/lib/apt/lists/*
-
-FROM archivebox-runtime-base AS archivebox-builder
-
-WORKDIR "$CODE_DIR"
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
-    --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    --mount=type=bind,source=pyproject.toml,target=/app/pyproject.toml \
-    echo "[+] UV Installing ArchiveBox dependencies from pyproject.toml..." \
-    && export UV_PYTHON_INSTALL_DIR="$ARCHIVEBOX_PYTHON_INSTALL_DIR" \
-    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
-    && echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-install-recommends \
-    && echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-install-suggests \
-    && rm -f /etc/apt/apt.conf.d/docker-clean \
-    && apt-get update -qq \
-    && apt-get install -qq -y --no-install-recommends \
-        build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && /usr/bin/uv venv --clear /venv --python "${PYTHON_VERSION}" \
-    && /usr/bin/uv pip install setuptools pip wheel \
-    && /usr/bin/uv sync \
-        --refresh \
-        --no-dev \
-        --inexact \
-        --all-extras \
-        --no-install-project \
-        --no-install-workspace \
-        --no-sources \
-    && apt-get purge -y build-essential gcc libldap2-dev libsasl2-dev libssl-dev \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
-RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$TARGETVARIANT \
-    echo "[*] Installing ArchiveBox Python source code from $CODE_DIR..." \
-    && COMMIT_HASH="$( \
-        if [[ -f "$CODE_DIR/.git/HEAD" ]]; then \
-            HEAD_REF="$(cat "$CODE_DIR/.git/HEAD")"; \
-            if [[ "$HEAD_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then \
-                echo "$HEAD_REF"; \
-            elif [[ "$HEAD_REF" == ref:\ * ]]; then \
-                REF_PATH="${HEAD_REF#ref: }"; \
-                cat "$CODE_DIR/.git/$REF_PATH" 2>/dev/null || awk -v ref="$REF_PATH" '$2 == ref {print $1}' "$CODE_DIR/.git/packed-refs" 2>/dev/null || true; \
-            fi; \
-        fi)" \
-    && if [[ "$COMMIT_HASH" =~ ^[0-9a-fA-F]{40}$ ]]; then echo "COMMIT_HASH=$COMMIT_HASH" | tee -a /VERSION.txt; fi \
-    && /usr/bin/uv pip install --no-deps "$CODE_DIR" \
-    && (/usr/bin/uv pip show archivebox && which archivebox) | tee -a /VERSION.txt \
-    && rm -rf "$CODE_DIR/.git"
-
-FROM archivebox-runtime-base
-
-LABEL name="archivebox" \
-    maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
-    description="All-in-one self-hosted internet archiving solution" \
-    homepage="https://github.com/ArchiveBox/ArchiveBox" \
-    documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker" \
-    org.opencontainers.image.title="ArchiveBox" \
-    org.opencontainers.image.vendor="ArchiveBox" \
-    org.opencontainers.image.description="All-in-one self-hosted internet archiving solution" \
-    org.opencontainers.image.source="https://github.com/ArchiveBox/ArchiveBox" \
-    com.docker.image.source.entrypoint="Dockerfile"
-
-COPY --from=sonic /usr/local/bin/sonic /usr/local/bin/sonic
-COPY --chown=root:root --chmod=755 "etc/sonic.cfg" /etc/sonic.cfg
-
-COPY --from=archivebox-builder /opt/archivebox/python /opt/archivebox/python
-COPY --from=archivebox-builder /venv /venv
-COPY --from=archivebox-builder /app /app
-COPY --from=archivebox-builder /VERSION.txt /VERSION.txt
-
-RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
-    && getent group "$ARCHIVEBOX_USER" >/dev/null || groupadd --system "$ARCHIVEBOX_USER" \
-    && id -u "$ARCHIVEBOX_USER" >/dev/null 2>&1 || useradd --system --create-home --gid "$ARCHIVEBOX_USER" --groups audio,video "$ARCHIVEBOX_USER" \
-    && usermod --append --groups audio,video "$ARCHIVEBOX_USER" \
-    && [[ "$(id -u "$ARCHIVEBOX_USER")" == "$DEFAULT_PUID" ]] || usermod -u "$DEFAULT_PUID" "$ARCHIVEBOX_USER" \
-    && [[ "$(id -g "$ARCHIVEBOX_USER")" == "$DEFAULT_PGID" ]] || groupmod -g "$DEFAULT_PGID" "$ARCHIVEBOX_USER" \
-    && (which sonic && sonic --version) | tee -a /VERSION.txt \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER" "/home/$ARCHIVEBOX_USER/.config" "/home/$ARCHIVEBOX_USER/.cache" \
-    && install -d -o "$DEFAULT_PUID" -g "$DEFAULT_PGID" "/home/$ARCHIVEBOX_USER/.config/abx" "/home/$ARCHIVEBOX_USER/.cache/abxbus" "/home/$ARCHIVEBOX_USER/.cache/pnpm" "/home/$ARCHIVEBOX_USER/.cache/uv" \
-    && chown "$DEFAULT_PUID:$DEFAULT_PGID" "$DATA_DIR" "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && openssl rand -hex 16 > /etc/machine-id \
-    && echo -e "\nARCHIVEBOX_USER=$ARCHIVEBOX_USER PUID=$(id -u "$ARCHIVEBOX_USER") PGID=$(id -g "$ARCHIVEBOX_USER")" | tee -a /VERSION.txt \
-    && echo -e "TMP_DIR=$TMP_DIR\nLIB_DIR=$LIB_DIR\nPLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH\nMACHINE_ID=$(cat /etc/machine-id)\n" | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-RUN echo "[+] Initializing image collection..." \
-    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
-    && PUID=0 PGID=0 archivebox init \
-    && (chown "$DEFAULT_PUID:$DEFAULT_PGID" \
-        "$DATA_DIR" "$DATA_DIR"/.archivebox_id "$DATA_DIR"/ArchiveBox.conf "$DATA_DIR"/index.sqlite3 \
-        "$DATA_DIR"/logs "$DATA_DIR"/logs/* "$DATA_DIR"/sources \
-        "$DATA_DIR"/archive "$DATA_DIR"/archive/users "$DATA_DIR"/personas \
-        "$DATA_DIR"/tmp "$DATA_DIR"/tmp/* \
-        "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-        2>/dev/null || true)
-
-RUN chmod +x "$CODE_DIR"/bin/*.sh \
-    && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
-
-RUN "$LIB_DIR/bin/chromium" --version | tee -a /VERSION.txt \
-    && "$LIB_DIR/uv/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
-    && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \
-    && /usr/local/bin/sonic --version | tee -a /VERSION.txt \
-    && /venv/bin/supervisord --version | tee -a /VERSION.txt \
-    && for forbidden_bin in gcc g++ make; do ! command -v "$forbidden_bin" || (echo "Unexpected build tool in runtime: $forbidden_bin=$(command -v "$forbidden_bin")" >&2 && exit 1); done \
-    && stat -c "%U:%G %a %n" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$LIB_DIR" \
-    && setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups archivebox version 2>&1 | tee -a /VERSION.txt \
-    && rm -rf /root/.cache /var/cache/apt/* /var/lib/apt/lists/*
-
-RUN (echo -e "\n\n[√] Finished ArchiveBox multistage Docker build successfully." \
-    && echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) (${TARGETARCH} ${TARGETVARIANT})" \
-    && echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
-    ) | tee -a /VERSION.txt
-
-WORKDIR "$DATA_DIR"
-VOLUME "$DATA_DIR"
-EXPOSE 8000
-
-HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
-    CMD curl --fail --silent --show-error --max-time 5 --connect-timeout 2 'http://admin.archivebox.localhost:8000/health/' | grep -q 'OK'
-
-ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
-CMD ["archivebox", "server", "--init", "0.0.0.0:8000"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 Nick Sweeting
+Copyright (c) 2026 Nick Sweeting
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -8,14 +8,8 @@
 
 <br/>
 
-<!--<a href="http://webchat.freenode.net?channels=ArchiveBox&uio=d4"><img src="https://img.shields.io/badge/Community_chat-IRC-%2328A745.svg"/></a>-->
-
 <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE"><img src="https://img.shields.io/badge/Open_source-MIT-green.svg?logo=git&logoColor=green"/></a> <a href="https://github.com/ArchiveBox/ArchiveBox/commits/dev"><img src="https://img.shields.io/github/last-commit/ArchiveBox/ArchiveBox.svg?logo=Sublime+Text&logoColor=green&label=Active"/></a> &nbsp; <a href="https://github.com/ArchiveBox/ArchiveBox"><img src="https://img.shields.io/github/stars/ArchiveBox/ArchiveBox.svg?logo=github&label=Stars&logoColor=blue"/></a> &nbsp; <a href="https://hub.docker.com/r/archivebox/archivebox"><img src="https://img.shields.io/docker/pulls/archivebox/archivebox.svg?label=Docker+Pulls"/></a> <a href="https://pypi.org/project/archivebox/"><img src="https://img.shields.io/pypi/dm/archivebox?label=PyPI%20Installs&color=%235f7dae"/></a> <a href="https://chromewebstore.google.com/detail/archivebox-exporter/habonpimjphpdnmcfkaockjnffodikoj"><img src="https://img.shields.io/chrome-web-store/users/habonpimjphpdnmcfkaockjnffodikoj?label=Chrome%20Store&color=%231973e8"/></a>
 
-<!--<pre lang="bash" align="left"><code style="white-space: pre-line; text-align: left" align="left">
-curl -fsSL 'https://get.archivebox.io' | bash    # (or see pip/brew/Docker instructions below)
-</code></pre>-->
-
 </div>
 <hr/>
 <br/>
@@ -29,7 +23,7 @@ Without active preservation effort, everything on the internet eventually disapp
 *ArchiveBox is an open source tool that lets organizations & individuals archive both public & private web content while retaining control over their data. It can be used to save copies of bookmarks, preserve evidence for legal cases, backup photos from FB/Insta/Flickr or media from YT/Soundcloud/etc., save research papers, and more...*
 <br/>
 
-> ➡️ Get ArchiveBox with `pip install archivebox` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.  
+> ➡️ Get ArchiveBox with `uv tool install --python 3.13 --prerelease explicit --upgrade 'archivebox>=0.9.0rc0,<0.10'` on [Linux](#quickstart)/[macOS](#quickstart), or via **[Docker](#quickstart)** ⭐️ on any OS.
 
 *Once installed, you can interact with it through the: [Browser Extension](https://github.com/ArchiveBox/archivebox-browser-extension), [CLI](#usage), [self-hosted web interface](https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive), [Python API](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage), or [filesystem](#static-archive-exporting).*
 
@@ -52,7 +46,7 @@ It also detects any content featured *inside* pages & extracts it out into a fol
 - 💾 **Github**/**Gitlab**/etc. links ➡️ `clone of GIT source code`, `README`, `images`, ...
 - ✨ *and more, see [Output Formats](#output-formats) below...*
 
-You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `pip`-installed CLI, Python API, and SQLite3 APIs. 
+You can run ArchiveBox as a Docker web app to manage these snapshots, or continue accessing the same collection using the `uv`-installed CLI, Python API, and SQLite3 APIs.
 All the ways of using it are equivalent, and provide matching features like adding tags, scheduling regular crawls, viewing logs, and more...
 
 <br/>
@@ -68,7 +62,7 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <br/>
 
 
-**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `pip` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
+**📦&nbsp; Install ArchiveBox using your preferred method: `docker` / `uv` / `brew` / `apt` / etc. ([see full Quickstart below](#quickstart)).**
 
 
 <details>
@@ -77,22 +71,24 @@ The goal is to sleep soundly knowing the part of the internet you care about wil
 <pre lang="bash"><code style="white-space: pre-line"># Option A: Get ArchiveBox with Docker Compose (recommended):
 mkdir -p ~/archivebox/data && cd ~/archivebox
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml   # edit options in this file as-needed
-docker compose run archivebox init
-# docker compose run archivebox add 'https://example.com'
-# docker compose run archivebox help
-# docker compose up
+docker compose pull
+docker compose up -d --wait                                                # initializes new collections automatically
+docker compose exec archivebox archivebox manage createsuperuser          # create the first Web UI user
+# docker compose run --rm archivebox add 'https://example.com'
+# docker compose run --rm archivebox help
 <br/>
 <br/>
 # Option B: Or use it as a plain Docker container:
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -it -v $PWD:/data archivebox/archivebox init
-# docker run -it -v $PWD:/data archivebox/archivebox add 'https://example.com'
-# docker run -it -v $PWD:/data archivebox/archivebox help
-# docker run -it -v $PWD:/data -p 8000:8000 archivebox/archivebox
+docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev init
+docker run --rm -it -v "$PWD:/data" archivebox/archivebox:dev manage createsuperuser
+docker run -d --name archivebox -v "$PWD:/data" -p 8000:8000 archivebox/archivebox:dev
+# docker run -it -v $PWD:/data archivebox/archivebox:dev add 'https://example.com'
+# docker run -it -v $PWD:/data archivebox/archivebox:dev help
 <br/>
 <br/>
-# Option C: Or install it with your preferred pkg manager (see Quickstart below for apt, brew, and more)
-pip install archivebox
+# Option C: Or install it with uv (see Quickstart below for apt, brew, and more)
+uv tool install --python 3.13 --prerelease explicit --upgrade 'archivebox>=0.9.0rc0,<0.10'
 mkdir -p ~/archivebox/data && cd ~/archivebox/data
 archivebox init
 archivebox install
@@ -106,7 +102,7 @@ curl -fsSL 'https://get.archivebox.io' | bash
 </code></pre>
 <br/>
 <sub>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sub><br/>
-<sub>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sub>
+<sub>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sub>
 </details>
 <br/>
 
@@ -171,7 +167,7 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 
 # Quickstart
 
-**🖥&nbsp; [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Linux/BSD, macOS, Windows (Docker) &nbsp; **👾&nbsp; CPUs:** `amd64` (`x86_64`), `arm64`, `arm7` <sup>(raspi>=3)</sup><br/>
+**🖥&nbsp; [Supported OSs](https://github.com/ArchiveBox/ArchiveBox/wiki/Install#supported-systems):** Ubuntu, macOS, Docker &nbsp; **👾&nbsp; CPUs:** `amd64` (`x86_64`), `arm64`<br/>
 
 <br/>
 
@@ -188,16 +184,17 @@ ArchiveBox is free for everyone to self-host, but we also provide support, secur
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox
 # Read and edit docker-compose.yml options as-needed after downloading
 curl -fsSL 'https://docker-compose.archivebox.io' > docker-compose.yml
+docker compose pull
 </code></pre></li>
-<li>Run the initial setup to create an admin user (or set ADMIN_USER/PASS in docker-compose.yml)
-<pre lang="bash"><code style="white-space: pre-line">docker compose run archivebox init
+<li>Start the server, which initializes a new collection automatically, then create the first admin user.
+<pre lang="bash"><code style="white-space: pre-line">docker compose up -d --wait
+docker compose exec archivebox archivebox manage createsuperuser
 </code></pre></li>
-<li>Next steps: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker compose up
-# completely optional, CLI can always be used without running a server
-# docker compose run [-T] archivebox [subcommand] [--help]
-docker compose run archivebox add 'https://example.com'
-docker compose run archivebox help
+<li>Next steps: Log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">
+# run CLI commands inside the server container started above
+docker compose exec archivebox archivebox add 'https://example.com'
+docker compose exec archivebox archivebox help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -214,14 +211,15 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <li>Install <a href="https://docs.docker.com/get-docker/">Docker</a> on your system (if not already installed).</li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-docker run -v $PWD:/data -it archivebox/archivebox init
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox:dev install
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox
+<li>Optional: Start the server then log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">docker run -v $PWD:/data -p 8000:8000 archivebox/archivebox:dev
 # completely optional, CLI can always be used without running a server
-# docker run -v $PWD:/data -it [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox help
+# docker run -v $PWD:/data -it archivebox/archivebox:dev [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
 </code></pre>
 <i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-a-docker--docker-compose-setup-%EF%B8%8F">Install: Docker Compose</a> in the Wiki. ➡️</i>
 </li>
@@ -256,27 +254,26 @@ See <a href="https://docs.sweeting.me/s/against-curl-sh">"Against curl | sh as a
 
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="Pip" height="28px" align="top"/> <code>pip</code></b> (macOS/Linux/BSD)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447613-ba4c5d80-af0b-11eb-8f89-1d98e31b6a79.png" alt="uv" height="28px" align="top"/> <code>uv</code></b> (macOS/Linux/BSD)</summary>
 <br/>
 <ol>
 
-<li>Install <a href="https://realpython.com/installing-python/">Python >= v3.13</a> and <a href="https://nodejs.org/en/download/package-manager/">Node >= v22</a> on your system (if not already installed).</li>
-<li>Install the ArchiveBox package using <code>pip3</code> (or <a href="https://docs.astral.sh/uv/guides/tools/#running-tools"><code>uvx</code></a>).
-<pre lang="bash"><code style="white-space: pre-line">pip3 install --upgrade archivebox
+<li>Install <a href="https://docs.astral.sh/uv/getting-started/installation/">uv</a> on your system (if not already installed).</li>
+<li>Install the ArchiveBox package using <code>uv</code>.
+<pre lang="bash"><code style="white-space: pre-line">uv tool install --python 3.13 --prerelease explicit --upgrade 'archivebox>=0.9.0rc0,<0.10'
 archivebox version
-# install any missing extras shown using apt/brew/pkg/etc. see Wiki for instructions
-#    python@3.13 node curl wget git ripgrep ...
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install">Install: Bare Metal</a> Wiki for full install instructions for each OS...</i>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # for example
-archivebox init     # instantialize a new collection
+archivebox init     # initialize a new collection
 archivebox install  # install all the runtime dependencies (e.g. chrome, single-file, yt-dlp, etc.)
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
+archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -286,7 +283,7 @@ archivebox help
 
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
 <br/>
-<sub>See the <a href="https://github.com/ArchiveBox/pip-archivebox"><code>pip-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>See the <a href="https://docs.astral.sh/uv/guides/tools/"><code>uv tool</code> documentation</a> for more details about this installation method.</sub>
 <br/><br/>
 </details>
 
@@ -299,17 +296,21 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <pre lang="bash"><code style="white-space: pre-line">echo 'deb [trusted=yes] https://archivebox.github.io/debian-archivebox dev main' | sudo tee /etc/apt/sources.list.d/archivebox.list
 sudo apt update
 sudo apt install archivebox
-archivebox version                         # make sure all dependencies are installed
+(cd /tmp && archivebox version)            # make sure the package is installed
 </code></pre>
 </li>
 <li>Create a new empty directory and initialize your collection (can be anywhere).
-<pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data
-archivebox init --install
+<pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data
+cd ~/archivebox/data
+archivebox init
+archivebox install
+archivebox add 'https://example.com'
 </code></pre>
 <br/>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
+archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -317,19 +318,21 @@ archivebox help
 </li>
 </ol>
 See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the CLI, Web UI, or filesystem/SQL/Python to manage your archive.<br/>
-<sub>See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for more details about this distribution.</sub>
+<sub>The apt package is a thin dev-channel wrapper around the normal Python install flow. See the <a href="https://github.com/ArchiveBox/debian-archivebox"><code>debian-archivebox</code></a> repo for details, and run <code>sudo archivebox install</code> only if you want it to install missing system packages via apt.</sub>
 <br/><br/>
 </details>
 
 <details>
-<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS only)</summary>
+<summary><b><img src="https://user-images.githubusercontent.com/511499/117447803-f2ec3700-af0b-11eb-87d3-671d114f011d.png" alt="homebrew" height="28px" align="top"/> <code>brew</code></b> (macOS and Linux)</summary>
 <br/>
+Run Homebrew as your normal non-root user on both macOS and Linux; do not use <code>sudo brew</code>.
 <ol>
 <li>Install <a href="https://brew.sh/#install">Homebrew</a> on your system (if not already installed).</li>
 <li>Install the ArchiveBox package using <code>brew</code>.
 <pre lang="bash"><code style="white-space: pre-line">brew tap archivebox/archivebox
+brew trust archivebox/archivebox
 brew install archivebox
-archivebox version                         # make sure all dependencies are installed
+archivebox version                         # verify the installed version
 </code></pre>
 <i>See the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup">Install: Bare Metal</a> Wiki for more granular instructions for macOS... ➡️</i>
 </li>
@@ -339,8 +342,9 @@ archivebox init
 archivebox install
 </code></pre>
 </li>
-<li>Optional: Start the server then login to the Web UI <a href="http://archivebox.localhost:8000">http://archivebox.localhost:8000</a> ⇢ Admin.
-<pre lang="bash"><code style="white-space: pre-line">archivebox server 0.0.0.0:8000
+<li>Create an admin account, then optionally start the server and log in to the Admin UI at <a href="http://admin.archivebox.localhost:8000">http://admin.archivebox.localhost:8000</a>.
+<pre lang="bash"><code style="white-space: pre-line">archivebox manage createsuperuser
+archivebox server 0.0.0.0:8000
 # completely optional, CLI can always be used without running a server
 # archivebox [subcommand] [--help]
 archivebox help
@@ -357,11 +361,11 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <summary><img src="https://user-images.githubusercontent.com/511499/118077361-f0616580-b381-11eb-973c-ee894a3349fb.png" alt="Arch" height="28px" align="top"/> <code>pacman</code> / <img src="https://user-images.githubusercontent.com/511499/118077946-29e6a080-b383-11eb-94f0-d4871da08c3f.png" alt="FreeBSD" height="28px" align="top"/> <code>pkg</code> / <img src="https://user-images.githubusercontent.com/511499/118077861-002d7980-b383-11eb-86a7-5936fad9190f.png" alt="Nix" height="28px" align="top"/> <code>nix</code> (Arch/FreeBSD/NixOS/more)</summary>
 <br/>
 
-> *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
+> *Warning: These are contributed by external volunteers and may lag behind the official `uv` and Docker channels.*
 
 <ul>
 <li>Arch: <a href="https://aur.archlinux.org/packages/archivebox/"><code>yay -S archivebox</code></a> (contributed by <a href="https://github.com/imlonghao"><code>@imlonghao</code></a>, maintained by <a href="https://github.com/jasongodev"><code>@jasongodev</code></a>)</li>
-<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>pip3</code> under-the-hood)</li>
+<li>FreeBSD: <a href="https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-easy-setup"><code>curl -fsSL 'https://get.archivebox.io' | bash</code></a> (uses <code>pkg</code> + <code>uv</code> under-the-hood)</li>
 <li>Nix: <a href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/misc/archivebox/default.nix"><code>nix-env --install archivebox</code></a> (contributed by <a href="https://github.com/siraben"><code>@siraben</code></a>)</li>
 <li>Guix: <a href="https://packages.guix.gnu.org/packages/archivebox/"><code>guix install archivebox</code></a> (contributed by <a href="https://github.com/rakino"><code>@rakino</code></a>)</li>
 <li>More: <a href="https://github.com/ArchiveBox/ArchiveBox/issues/new"><i>contribute another distribution...!</i></a></li>
@@ -397,7 +401,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
 <summary><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/0c46e949-00fe-49c8-a613-ee14501c014c" alt="Self-hosting Platforms" height="28px" align="top"/><b> TrueNAS / UNRAID / YunoHost / Cloudron / etc.</b> (self-hosting solutions)</summary>
 <br/>
 
-> *Warning: These are contributed by external volunteers and may lag behind the official `pip` channel.*
+> *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
 
 <ul>
 <li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
@@ -482,11 +486,11 @@ cd ~/archivebox/data         # IMPORTANT: cd into the directory
 archivebox version
 archivebox help
 
-# equivalent: docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox help
+# equivalent: docker compose run --rm archivebox [subcommand] [--help]
+docker compose run --rm archivebox help
 
-# equivalent: docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -it -v $PWD:/data archivebox/archivebox help
+# equivalent: docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -it -v $PWD:/data archivebox/archivebox:dev help
 
 # optional: import your browser cookies into a persona for logged-in archiving
 archivebox persona create --import=chrome personal
@@ -507,13 +511,15 @@ archivebox persona create --import=chrome personal
 <summary><img src="https://user-images.githubusercontent.com/511499/117456282-08665e80-af16-11eb-91a1-8102eff54091.png" alt="curl sh automatic setup script" height="22px" align="top"/> <b>CLI Usage Examples: non-Docker</b></summary>
 <br/>
 <pre lang="bash"><code style="white-space: pre-line">
-# make sure you have pip-installed ArchiveBox and it's available in your $PATH first  
+# make sure you have uv-installed ArchiveBox and it's available in your $PATH first
 <br/>
 # archivebox [subcommand] [--help]
-archivebox init --install      # safe to run init multiple times (also how you update versions)
+mkdir -p ~/archivebox/data && cd ~/archivebox/data
+archivebox init
+archivebox install
+archivebox add 'https://example.com'
 archivebox version           # get archivebox version info + check dependencies
 archivebox help              # get list of archivebox subcommands that can be run
-archivebox add --depth=1 'https://news.ycombinator.com'
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage">Usage: CLI Usage</a> wiki. ➡️</i>
 </details>
@@ -526,11 +532,12 @@ archivebox add --depth=1 'https://news.ycombinator.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you have `docker-compose.yml` from the Quickstart instructions first
 <br/>
-# docker compose run archivebox [subcommand] [--help]
-docker compose run archivebox init --install
-docker compose run archivebox version
-docker compose run archivebox help
-docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
+# docker compose run --rm archivebox [subcommand] [--help]
+docker compose run --rm archivebox init
+docker compose run --rm archivebox install
+docker compose run --rm archivebox version
+docker compose run --rm archivebox help
+docker compose run --rm archivebox add 'https://example.com'
 # to start webserver: docker compose up
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage">Usage: Docker Compose CLI</a> wiki. ➡️</i>
@@ -544,12 +551,13 @@ docker compose run archivebox add --depth=1 'https://news.ycombinator.com'
 <pre lang="bash"><code style="white-space: pre-line">
 # make sure you create and cd into in a new empty directory first  
 <br/>
-# docker run -it -v $PWD:/data archivebox/archivebox [subcommand] [--help]
-docker run -v $PWD:/data -it archivebox/archivebox init --install
-docker run -v $PWD:/data -it archivebox/archivebox version
-docker run -v $PWD:/data -it archivebox/archivebox help
-docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.ycombinator.com'
-# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+# docker run -it -v $PWD:/data archivebox/archivebox:dev [subcommand] [--help]
+docker run -v $PWD:/data -it archivebox/archivebox:dev init
+docker run -v $PWD:/data -it archivebox/archivebox:dev install
+docker run -v $PWD:/data -it archivebox/archivebox:dev version
+docker run -v $PWD:/data -it archivebox/archivebox:dev help
+docker run -v $PWD:/data -it archivebox/archivebox:dev add 'https://example.com'
+# to start webserver: docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#usage-1">Usage: Docker CLI</a> wiki. ➡️</i>
 </details>
@@ -561,7 +569,7 @@ docker run -v $PWD:/data -it archivebox/archivebox add --depth=1 'https://news.y
 <pre lang="bash"><code style="white-space: pre-line">
 archivebox shell           # explore the Python library API in a REPL
 sqlite3 ./index.sqlite3    # run SQL queries directly on your index
-ls ./archive/*/index.html  # or inspect snapshot data directly on the filesystem
+find ./archive/users -path '*/snapshots/*/*/*/index.html'  # inspect snapshot data directly
 </code></pre>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage">Python Shell</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage">SQL API</a>, and <a href="https://github.com/ArchiveBox/ArchiveBox#archive-layout">Disk Layout</a> wikis. ➡️</i>
 </details>
@@ -572,21 +580,21 @@ ls ./archive/*/index.html  # or inspect snapshot data directly on the filesystem
 <details>
 <summary><b>🖥&nbsp; Web UI & API Usage</b></summary>
 <pre lang="bash"><code style="white-space: pre-line">
-# Start the server on bare metal (pip/apt/brew/etc):
+# Start the server on bare metal (uv/apt/brew):
 archivebox manage createsuperuser              # create a new admin user via CLI
 archivebox server 0.0.0.0:8000                 # start the server
 <br/>
 # Or with Docker Compose:
-nano docker-compose.yml                        # setup initial ADMIN_USERNAME & ADMIN_PASSWORD
+docker compose run --rm archivebox manage createsuperuser
 docker compose up                              # start the server
 <br/>
 # Or with a Docker container:
-docker run -v $PWD:/data -it archivebox/archivebox archivebox manage createsuperuser
-docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
+docker run -v $PWD:/data -it archivebox/archivebox:dev manage createsuperuser
+docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox:dev
 </code></pre>
 
 <sup>Open <a href="http://web.archivebox.localhost:8000"><code>http://web.archivebox.localhost:8000</code></a> for the public UI and <a href="http://admin.archivebox.localhost:8000"><code>http://admin.archivebox.localhost:8000</code></a> for the admin UI ➡️</sup><br/>
-<sup>Set <code>BIND_ADDR</code> to change the base domain; <code>web.</code> and <code>admin.</code> subdomains are used automatically.</sup>
+<sup>Set <code>BASE_URL</code> to change the public base domain. The default <code>auto</code> mode uses <code>web.</code> and <code>admin.</code> subdomains on <code>*.localhost</code>, but one host for ordinary DNS names. <code>BIND_ADDR</code> only controls the local listen address.</sup>
 <br/><br/>
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage">Usage: Web UI</a> wiki. ➡️</i>
 <br/><br/>
@@ -594,10 +602,10 @@ docker run -v $PWD:/data -it -p 8000:8000 archivebox/archivebox
 
 <pre lang="bash"><code style="white-space: pre-line">
 archivebox config --set PUBLIC_ADD_VIEW=True   # allow guests to submit URLs 
-archivebox config --set PUBLIC_SNAPSHOTS=True  # allow guests to see snapshot content
+archivebox config --set PERMISSIONS=public     # make newly added snapshots public
 archivebox config --set PUBLIC_INDEX=True      # allow guests to see list of all snapshots
 # or
-docker compose run archivebox config --set ...
+docker compose run --rm archivebox config --set PERMISSIONS=public
 
 # restart the server to apply any config changes
 </code></pre>
@@ -608,15 +616,15 @@ docker compose run archivebox config --set ...
 
 > [!TIP]
 > Whether in Docker or not, ArchiveBox commands work the same way, and can be used to access the same data on-disk.
-> For example, you could run the Web UI in Docker Compose, and run one-off commands with `pip`-installed ArchiveBox.
+> For example, you could run the Web UI in Docker Compose, and run one-off commands with `uv`-installed ArchiveBox.
 
 <details>
 <summary><i>Expand to show comparison...</i></summary><br/>
 
 <pre lang="bash"><code style="white-space: pre-line">
-archivebox add --depth=1 'https://example.com'                     # add a URL with pip-installed archivebox on the host
-docker compose run archivebox add --depth=1 'https://example.com'                       # or w/ Docker Compose
-docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
+archivebox add --depth=1 'https://example.com'                     # add a URL with uv-installed archivebox on the host
+docker compose run --rm archivebox add --depth=1 'https://example.com'                  # or w/ Docker Compose
+docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://example.com'  # or w/ Docker, all equivalent
 </code></pre>
 
 <i>For more info, see our <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker">Docker</a> wiki. ➡️</i>
@@ -677,15 +685,15 @@ docker run -it -v $PWD:/data archivebox/archivebox add --depth=1 'https://exampl
 ```bash
 # archivebox add --help
 archivebox add 'https://example.com/some/page'
-archivebox add --parser=generic_rss < ~/Downloads/some_feed.xml
+archivebox add --depth=1 --plugins=parse_rss_urls "file://$HOME/Downloads/some_feed.xml"
 archivebox add --depth=1 'https://news.ycombinator.com#2020-12-12'
 echo 'http://example.com' | archivebox add
 echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox add
 
 # if using Docker, add -i when piping stdin:
-# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox add
+# echo 'https://example.com' | docker run -v $PWD:/data -i archivebox/archivebox:dev add
 # if using Docker Compose, add -T when piping stdin / stdout:
-# echo 'https://example.com' | docker compose run -T archivebox add
+# echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
 See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
@@ -743,10 +751,10 @@ ArchiveBox can be configured via environment variables, by using the `archivebox
 archivebox config --get CHROME_BINARY           # view a specific value
 <br/>
 archivebox config --set CHROME_BINARY=chromium  # persist a config using CLI
+# OR edit ArchiveBox.conf and add this under its existing [ARCHIVING_CONFIG] section:
+CHROME_BINARY=chromium
 # OR
-echo CHROME_BINARY=chromium >> ArchiveBox.conf  # persist a config using file
-# OR
-env CHROME_BINARY=chromium archivebox ...       # run with a one-off config
+env CHROME_BINARY=chromium archivebox version   # run with a one-off config
 </code></pre>
 <sub>These methods also work the same way when run inside Docker, see the <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker Configuration</a> wiki page for details.</sub>
 </details><br/>
@@ -758,13 +766,13 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
 <summary><i>Expand to see the most common options to tweak...</i></summary>
 <pre lang="bash"><code style="white-space: pre-line">
 # e.g. archivebox config --set TIMEOUT=120
-# or   docker compose run archivebox config --set TIMEOUT=120
+# or   docker compose run --rm archivebox config --set TIMEOUT=120
 <br/>
 TIMEOUT=240                # default: 60    add more seconds on slower networks
 CHECK_SSL_VALIDITY=False   # default: True  False = allow saving URLs w/ bad SSL
 <br/>
 PUBLIC_INDEX=True          # default: True  whether anon users can view index
-PUBLIC_SNAPSHOTS=True      # default: True  whether anon users can view pages
+PERMISSIONS=public         # default: public visibility for newly added snapshots
 PUBLIC_ADD_VIEW=False      # default: False whether anon users can add new URLs
 <br/>
 USER_AGENT="Mozilla/5.0 ..."  # change this to get around bot blocking
@@ -817,14 +825,10 @@ These optional subdependencies used for archiving sites include:
 <li>and more as we grow...</li>
 </ul>
 
-You don't need to install every dependency to use ArchiveBox. ArchiveBox will automatically disable extractors that rely on dependencies that aren't installed, based on what is configured and available in your <code>$PATH</code>.
-  
-If not using Docker, make sure to keep the dependencies up-to-date yourself and check that ArchiveBox isn't reporting any incompatibility with the versions you install.
+You don't need to install every dependency by hand. ArchiveBox resolves every extractor dependency through <code>abxpkg</code>: it uses a compatible host installation when one is already available, and otherwise installs and manages the dependency for you.
 
-<pre lang="bash"><code style="white-space: pre-line">#install python3 and archivebox with your system package manager
-# apt/brew/pip/etc install ... (see Quickstart instructions above)
+<pre lang="bash"><code style="white-space: pre-line"># install uv + archivebox first (see Quickstart instructions above)
 <br/>
-which -a archivebox    # see where you have installed archivebox
 archivebox install     # auto install all the extractors and extras
 archivebox --version   # see info and check validity of installed dependencies
 </code></pre>
@@ -857,7 +861,7 @@ All <code>archivebox</code> CLI commands are designed to be run from inside an A
 <pre lang="bash"><code style="white-space: pre-line">mkdir -p ~/archivebox/data && cd ~/archivebox/data   # just an example, can be anywhere
 archivebox init</code></pre>
 
-The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard <code>index.sqlite3</code> database in the root of the data folder (it can also be <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">exported as static JSON/HTML</a>), and the archive snapshots are organized by date-added timestamp in the <code>data/archive/</code> subfolder.
+The on-disk layout is optimized to be easy to browse by hand and durable long-term. The main index is a standard <code>index.sqlite3</code> database in the root of the data folder (it can also be <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">exported as static JSON/HTML</a>). Snapshot data is organized by user, date, domain, and UUID under <code>data/archive/users/</code>.
 
 <img src="https://user-images.githubusercontent.com/511499/117453293-c7b91600-af12-11eb-8a3f-aa48b0f9da3c.png" width="400px" align="right" style="float: right"/>
 
@@ -866,18 +870,23 @@ The on-disk layout is optimized to be easy to browse by hand and durable long-te
     index.sqlite3
     ArchiveBox.conf
     archive/
-        ...
-        1617687755/
-            index.html
-            index.json
-            screenshot.png
-            media/some_video.mp4
-            warc/1617687755.warc.gz
-            git/somerepo.git
-            ...
+        1617687755 -> users/admin/snapshots/20210406/example.com/SNAPSHOT_UUID/
+        users/
+            admin/
+                snapshots/
+                    20210406/
+                        example.com/
+                            SNAPSHOT_UUID/
+                                index.html
+                                index.jsonl
+                                screenshot/screenshot.png
+                                ytdlp/media/some_video.mp4
+                                wget/warc/example.com.warc.gz
+                                git/somerepo.git
+                                ...
 </code></pre>
 
-Each snapshot subfolder <code>data/archive/TIMESTAMP/</code> includes a static <code>index.json</code> and <code>index.html</code> describing its contents, and the snapshot extractor outputs are plain files within the folder.
+Each snapshot subfolder includes static metadata and plain extractor output files. ArchiveBox also maintains a backwards-compatible <code>data/archive/TIMESTAMP</code> symlink for each snapshot.
 
 <h4>Learn More</h4>
 <ul>
@@ -911,7 +920,7 @@ archivebox list --json --with-headers > index.json     # export to json blob
 archivebox list --csv=timestamp,url,title > index.csv  # export to csv spreadsheet
 
 # (if using Docker Compose, add the -T flag when piping)
-# docker compose run -T archivebox list --html 'https://example.com' > index.json
+# docker compose run --rm -T archivebox list --html 'https://example.com' > index.html
 </code></pre>
 
 The paths in the static exports are relative, make sure to keep them next to your `./archive` folder when backing them up or viewing them.
@@ -921,7 +930,7 @@ The paths in the static exports are relative, make sure to keep them next to you
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive#2-export-and-host-it-as-static-html">Wiki: Publishing Your Archive (Exporting as Static HTML)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#publishing">Wiki: Security Overview (Publishing)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#public_index--public_snapshots--public_add_view">Wiki: Configuration (<code>PUBLIC_INDEX</code>, <code>PUBLIC_SNAPSHOTS</code>, <code>PUBLIC_ADD_VIEW</code>)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions">Wiki: Configuration (<code>PERMISSIONS</code>, <code>PUBLIC_INDEX</code>, <code>PUBLIC_ADD_VIEW</code>)</a></li>
 </ul>
 
 </details>
@@ -952,7 +961,7 @@ archivebox add 'https://vimeo.com/somePrivateVideo'
 
 # restrict the main index, Snapshot content, and Add Page to authenticated users as-needed:
 archivebox config --set PUBLIC_INDEX=False
-archivebox config --set PUBLIC_SNAPSHOTS=False
+archivebox config --set PERMISSIONS=private
 archivebox config --set PUBLIC_ADD_VIEW=False
 archivebox manage createsuperuser
 </code></pre>
@@ -968,8 +977,8 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir">Wiki: Configuration (<code>CHROME_USER_DATA_DIR</code>)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#cookies_file">Wiki: Configuration (<code>COOKIES_FILE</code>)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">Wiki: Personas (browser profiles and cookies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
 </details>
@@ -978,7 +987,7 @@ archivebox manage createsuperuser
 
 ### Security Risks of Viewing Archived JS
 
-Be aware that malicious archived JS can access the contents of other pages in your archive when viewed. Because the Web UI serves all viewed snapshots from a single domain, they share a request context and **typical CSRF/CORS/XSS/CSP protections do not work to prevent cross-site request attacks**. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) page and [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239) for more details.
+Archived JavaScript is untrusted content. The default <code>SERVER_SECURITY_MODE=auto</code> uses isolated subdomains with full replay on <code>*.localhost</code>, and a one-domain no-JS replay policy on ordinary public or LAN hostnames. Choose <code>safe-subdomains-fullreplay</code> only when wildcard DNS and TLS are configured. See the [Security Overview](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview) and [Issue #239](https://github.com/ArchiveBox/ArchiveBox/issues/239) for details.
 
 
 <br/>
@@ -986,18 +995,16 @@ Be aware that malicious archived JS can access the contents of other pages in yo
 <summary><i>Expand to see risks and mitigations...</i></summary>
 
 
-<pre lang="bash"><code style="white-space: pre-line"># visiting an archived page with malicious JS:
-https://127.0.0.1:8000/archive/1602401954/example.com/index.html
+<pre lang="bash"><code style="white-space: pre-line"># Explicit wildcard mode: full replay on isolated snapshot subdomains
+archivebox config --set SERVER_SECURITY_MODE=safe-subdomains-fullreplay
 
-# example.com/index.js can now make a request to read everything from:
-https://127.0.0.1:8000/index.html
-https://127.0.0.1:8000/archive/*
-# then example.com/index.js can send it off to some evil server
+# Alternative for deployments without wildcard subdomains: disable JS replay
+archivebox config --set SERVER_SECURITY_MODE=safe-onedomain-nojsreplay
 </code></pre>
 
 <blockquote>
 <p><em>NOTE: Only the <code>wget</code> &amp; <code>dom</code> extractor methods execute archived JS when viewing snapshots, all other archive methods produce static output that does not execute JS on viewing.</em><br/>
-<em>If you are worried about these issues ^ you should disable these extractors using:<br/> <code>archivebox config --set SAVE_WGET=False SAVE_DOM=False</code>.</em></p>
+<em>If you do not need JavaScript-capable replay at all, you can also disable those extractors with:<br/> <code>archivebox config --set WGET_ENABLED=False DOM_ENABLED=False</code>.</em></p>
 </blockquote>
 
 <h4>Learn More</h4>
@@ -1024,7 +1031,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 <ul>
 <li>Set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#curl_user_agent"><code>CHROME_USER_AGENT</code>, <code>WGET_USER_AGENT</code>, <code>CURL_USER_AGENT</code></a> to impersonate a real browser (by default, ArchiveBox reveals that it's a bot when using the default user agent settings)</li>
-<li>Set up a logged-in browser session for archiving using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile"><code>CHROME_USER_DATA_DIR</code> &amp; <code>COOKIES_FILE</code></a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Personas">importing a dedicated browser profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1037,7 +1044,7 @@ In the future we plan on adding support for running JS scripts during archiving
 
 ### Saving Multiple Snapshots of a Single URL
 
-ArchiveBox appends a hash with the current date `https://example.com#2020-10-24` to differentiate when a single URL is archived multiple times.
+ArchiveBox can preserve multiple snapshots of the same URL. The default <code>ONLY_NEW=True</code> skips URLs already in the collection; use <code>--no-only-new</code> when you intentionally want another snapshot.
 
 
 <br/>
@@ -1046,16 +1053,13 @@ ArchiveBox appends a hash with the current date `https://example.com#2020-10-24`
 <br/>
 
 
-Because ArchiveBox uniquely identifies snapshots by URL, it must use a workaround to take multiple snapshots of the same URL (otherwise they would show up as a single Snapshot entry). It makes the URLs of repeated snapshots unique by adding a hash with the archive date at the end:
+Each re-archive creates a distinct Snapshot row for the same URL:
 
-<pre lang="bash"><code style="white-space: pre-line">archivebox add 'https://example.com#2020-10-24'
-...
-archivebox add 'https://example.com#2020-10-25'
+<pre lang="bash"><code style="white-space: pre-line">archivebox add 'https://example.com'
+archivebox add --no-only-new 'https://example.com'
 </code></pre>
 
-The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI is a shortcut for this hash-date multi-snapshotting workaround.
-
-Improved support for saving multiple snapshots of a single URL without this hash-date workaround will be <a href="https://github.com/ArchiveBox/ArchiveBox/issues/179">added eventually</a> (along with the ability to view diffs of the changes between runs).
+The <img src="https://user-images.githubusercontent.com/511499/115942091-73c02300-a476-11eb-958e-5c1fc04da488.png" alt="Re-Snapshot Button" height="24px"/> button in the Admin UI performs the same explicit re-archive.
 
 <h4>Learn More</h4>
 
@@ -1083,7 +1087,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.
 </li>
 <li><strong>Try to keep the <code>data/index.sqlite3</code> file on local drive (not a network mount)</strong> or SSD for maximum performance, however the <code>data/archive/</code> folder can be on a network mount or slower HDD.</li>
-<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, you may need to set <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid"><code>PUID</code> & <code>PGID</code></a> and <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304">disable <code>root_squash</code></a> on your fileshare server.
+<li>If using Docker or NFS/SMB/FUSE for the <code>data/archive/</code> folder, make sure the mounted data directory is writable by its intended owner and consider disabling <a href="https://github.com/ArchiveBox/ArchiveBox/issues/1304"><code>root_squash</code></a> on your fileshare server.
 </li>
 </ul>
 
@@ -1093,8 +1097,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#puid--pgid">Wiki: Configuration (<code>PUID</code> & <code>GUID</code>)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root">Wiki: Security Overview (Do Not Run as Root)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
 </ul>
 
 
@@ -1183,7 +1186,7 @@ ArchiveBox's stance is that duplication of other people's content is only ethica
 
 In the U.S., <a href="https://guides.library.oregonstate.edu/copyright/libraries">libraries, researchers, and archivists</a> are allowed to duplicate copyrighted materials under <a href="https://libguides.ala.org/copyright/fairuse">"fair use"</a> for <a href="https://guides.cuny.edu/cunyfairuse/librarians#:~:text=One%20of%20these%20specified%20conditions,may%20be%20liable%20for%20copyright">private study, scholarship, or research</a>. Archive.org's non-profit preservation work is <a href="https://blog.archive.org/2024/03/01/fair-use-in-action-at-the-internet-archive/">covered under fair use</a> in the US, and they properly handle <a href="https://cardozoaelj.com/2015/03/20/use-of-copyright-law-to-take-down-revenge-porn/">unethical content</a>/<a href="https://help.archive.org/help/rights/">DMCA</a>/<a href="https://gdpr.eu/right-to-be-forgotten/#:~:text=An%20individual%20has%20the%20right,that%20individual%20withdraws%20their%20consent.">GDPR</a> removal requests to maintain good standing in the eyes of the law.
 
-As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal council if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and changing your instance's branding using <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#custom_templates_dir"><code>CUSTOM_TEMPLATES_DIR</code></a>).
+As long as you A. don't try to profit off pirating copyrighted content and B. have processes in place to respond to removal requests, many countries allow you to use software like ArchiveBox to ethically and responsibly archive any web content you can view. That being said, ArchiveBox is not liable for how you choose to operate the software. You must research your own local laws and regulations, and get proper legal counsel if you plan to host a public instance (start by putting your DMCA/GDPR contact info in <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#footer_info"><code>FOOTER_INFO</code></a> and placing branding overrides in your collection's fixed <code>custom_templates/</code> directory).
 
 </details>
 <br/>
@@ -1233,8 +1236,6 @@ ArchiveBox is neither the highest fidelity nor the simplest tool available for s
 
 <br/>
 
-<!--<div align="center" style="text-align: center"><br/><img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/04808ac2-3133-44fd-8703-3387e06dc851" width="100%" alt="dependencies graphic"></div>-->
-
 ## Internet Archiving Ecosystem
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/78d8a725-97f4-47f5-b983-1f62843ddc51" width="14%" align="right" style="float: right"/>
@@ -1333,7 +1334,7 @@ All contributions to ArchiveBox are welcomed! Check our [issues](https://github.
 
 For low hanging fruit / easy first tickets, see: <a href="https://github.com/ArchiveBox/ArchiveBox/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22help+wanted%22">ArchiveBox/Issues `#good first ticket` `#help wanted`</a>.
 
-**Python API Documentation:** https://docs.archivebox.io/en/dev/archivebox.html#module-archivebox.main
+**Python API Documentation:** https://docs.archivebox.io/dev/apidocs/
 
 **Internal Architecture Diagrams:** https://github.com/ArchiveBox/ArchiveBox/wiki/ArchiveBox-Architecture-Diagrams
 
@@ -1388,7 +1389,8 @@ archivebox server 0.0.0.0:8000
 # inside the container will reload and pick up your changes
 ./bin/build_docker.sh dev
 
-docker run -it -v $PWD/data:/data archivebox/archivebox:dev init --install
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev init
+docker run -it -v $PWD/data:/data archivebox/archivebox:dev install
 
 # Run the development server w/ autoreloading (but no bg workers)
 docker run -it -v $PWD/data:/data -v $PWD/archivebox:/app/archivebox -p 8000:8000 archivebox/archivebox:dev server --debug --reload 0.0.0.0:8000
@@ -1442,7 +1444,7 @@ docker run archivebox/archivebox:dev version
 ```
 
 ##### Build Branch from Source
-  
+
 You can also build and run any branch yourself from source, for example to build & use `dev` locally:
 
 ```bash
@@ -1457,9 +1459,8 @@ services:
 docker build -t archivebox:dev https://github.com/ArchiveBox/ArchiveBox.git#dev
 docker run -it -v $PWD:/data archivebox:dev init
 
-# or with pip:
-pip install 'git+https://github.com/pirate/ArchiveBox@dev'
-npm install 'git+https://github.com/ArchiveBox/ArchiveBox.git#dev'
+# or with uv:
+uv tool install --python 3.13 --upgrade 'git+https://github.com/ArchiveBox/ArchiveBox.git@dev'
 archivebox install
 ```
 
@@ -1484,8 +1485,7 @@ archivebox install
 
 ```bash
 # generate the database migrations after changes to models.py
-cd archivebox/
-./manage.py makemigrations
+archivebox manage makemigrations
 
 # enter a python shell or a SQL shell
 cd path/to/test/data/
@@ -1494,8 +1494,7 @@ archivebox manage dbshell
 
 # generate a graph of the ORM models
 brew install graphviz
-pip install pydot graphviz
-archivebox manage graph_models -a -o orm.png
+uv run --with pydot --with graphviz archivebox manage graph_models -a -o orm.png
 open orm.png
 
 # list all models with field db info and methods
@@ -1503,7 +1502,7 @@ archivebox manage list_model_info --all --signature --db-type --field-class
 
 # print all django settings
 archivebox manage print_settings
-archivebox manage print_settings --format=yaml    # pip install pyyaml
+uv run --with pyyaml archivebox manage print_settings --format=yaml
 
 # autogenerate an admin.py from given app models
 archivebox manage admin_generator core > core/admin.py
@@ -1517,9 +1516,8 @@ archivebox manage runscript testdata
 archivebox manage reset_db
 
 # use django-tui to interactively explore commands
-uv pip install django-tui
 # ensure django-tui is in INSTALLED_APPS: core/settings.py
-archivebox manage tui
+uv run --with django-tui archivebox manage tui
 ```
 
 <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/dc3e9f8c-9544-46e0-a7f0-30f571b72022" width="600px" alt="ArchiveBox ORM models relatinoship graph"/>
@@ -1557,22 +1555,6 @@ Copy a similar plugin as a template to modify, then open a new PR to add it in t
 
 </details>
 
-#### Roll a release
-
-<details><summary><i>Click to expand...</i></summary>
-
-(Normally CI takes care of this, but these scripts can be run to do it manually)
-```bash
-./bin/release.sh
-
-# or individually:
-./bin/release_docs.sh
-./bin/release_pip.sh
-./bin/release_docker.sh
-```
-
-</details>
-
 ---
 
 ## Further Reading

archivebox/__init__.py

@@ -38,11 +38,6 @@ ASCII_LOGO = """
 
 PACKAGE_DIR = Path(__file__).resolve().parent
 
-# # Add PACKAGE_DIR to sys.path - required for Django migrations to import models
-# # Migrations reference models like 'machine.Binary' which need to be importable
-# if str(PACKAGE_DIR) not in sys.path:
-#     sys.path.append(str(PACKAGE_DIR))
-
 os.environ["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
 os.environ["TZ"] = "UTC"
 
@@ -51,10 +46,9 @@ from .config.permissions import drop_privileges  # noqa
 
 drop_privileges()
 
-from .misc.checks import check_not_root, check_not_inside_source_dir, check_io_encoding  # noqa
+from .misc.checks import check_not_root, check_io_encoding  # noqa
 
 check_not_root()
-check_not_inside_source_dir()
 check_io_encoding()
 
 from .config.version import VERSION  # noqa

archivebox/api/migrations/0001_initial.py

@@ -12,19 +12,9 @@ import signal_webhooks.fields
 import signal_webhooks.utils
 
 
-class Migration(migrations.Migration):
-    initial = True
-
-    dependencies = [
-        ("auth", "0012_alter_user_first_name_max_length"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                migrations.RunSQL(
-                    sql="""
+# Raw sqlite DDL kept EXACTLY as-is; executed only on sqlite via Django's
+# RunSQL (identical statement splitting → byte-for-byte identical behavior).
+INITIAL_SQL = """
                 -- Create api_apitoken table
                 CREATE TABLE IF NOT EXISTS api_apitoken (
                     id TEXT PRIMARY KEY NOT NULL,
@@ -70,12 +60,60 @@ class Migration(migrations.Migration):
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_at_idx ON api_outboundwebhook(created_at);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
                 CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
-                    """,
-                    reverse_sql="""
+                    """
+
+INITIAL_REVERSE_SQL = """
                 DROP TABLE IF EXISTS api_outboundwebhook;
                 DROP TABLE IF EXISTS api_apitoken;
-                    """,
-                ),
+                    """
+
+
+def _run_sqlite_only_sql(apps, schema_editor):
+    if schema_editor.connection.vendor != "sqlite":
+        return
+    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_forwards(
+        "api",
+        schema_editor,
+        None,
+        None,
+    )
+
+
+def _run_sqlite_only_sql_reverse(apps, schema_editor):
+    if schema_editor.connection.vendor != "sqlite":
+        return
+    migrations.RunSQL(sql=INITIAL_SQL, reverse_sql=INITIAL_REVERSE_SQL).database_backwards(
+        "api",
+        schema_editor,
+        None,
+        None,
+    )
+
+
+def _pg_sync_schema(apps, schema_editor):
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "api", ["APIToken", "OutboundWebhook"])
+
+
+def _pg_drop_schema(apps, schema_editor):
+    from archivebox.misc.db import drop_models_on_postgres
+
+    drop_models_on_postgres(apps, schema_editor, "api", ["OutboundWebhook", "APIToken"])
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                migrations.RunPython(_run_sqlite_only_sql, reverse_code=_run_sqlite_only_sql_reverse),
             ],
             state_operations=[
                 migrations.CreateModel(
@@ -236,4 +274,8 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        # On non-sqlite backends the raw DDL above is skipped, so the real
+        # schema diverges from Django state. Rebuild both tables (including the
+        # unique constraint) from the post-migration state (no-op on sqlite).
+        migrations.RunPython(_pg_sync_schema, reverse_code=_pg_drop_schema),
     ]

archivebox/api/urls.py

@@ -23,7 +23,4 @@ urlpatterns = [
     path("v1/", RedirectView.as_view(url="/api/v1/docs")),
     path("v1/", v1_api_urls),
     path("v1", RedirectView.as_view(url="/api/v1/docs")),
-    # ... v2 can be added here ...
-    # path("v2/",              v2_api_urls),
-    # path("v2",               RedirectView.as_view(url='/api/v2/docs')),
 ]

archivebox/api/v1_api.py

@@ -78,9 +78,6 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
-        # import ipdb; ipdb.set_trace()
-        # print('RESPONDING NOW', response)
-
         return response
 
 
@@ -91,8 +88,6 @@ api = NinjaAPIWithIOCapture(
     auth=API_AUTH_METHODS,
     urls_namespace="api-1",
     docs=Swagger(settings={"persistAuthorization": True}),
-    # docs_decorator=login_required,
-    # renderer=ORJSONRenderer(),
 )
 api = register_urls(api)
 urls = api.urls
@@ -113,24 +108,7 @@ def generic_exception_handler(request, err):
             "message": f"{err.__class__.__name__}: {err}",
             "errors": [
                 "".join(format_exception(err)),
-                # or send simpler parent-only traceback:
-                # *([str(err.__context__)] if getattr(err, '__context__', None) else []),
             ],
         },
         status=status,
     )
-
-
-# import orjson
-# from ninja.renderers import BaseRenderer
-# class ORJSONRenderer(BaseRenderer):
-#     media_type = "application/json"
-#     def render(self, request, data, *, response_status):
-#         return {
-#             "success": True,
-#             "errors": [],
-#             "result": data,
-#             "stdout": ansi_to_html(stdout.getvalue().strip()),
-#             "stderr": ansi_to_html(stderr.getvalue().strip()),
-#         }
-#         return orjson.dumps(data)

archivebox/api/v1_cli.py

@@ -15,8 +15,6 @@ from archivebox.misc.util import ansi_to_html
 from archivebox.core.models import SnapshotQuerySet
 
 
-# from .auth import API_AUTH_METHODS
-
 # router for API that exposes archivebox cli subcommands as REST endpoints
 router = Router(tags=["ArchiveBox CLI Sub-Commands"])
 
@@ -54,6 +52,7 @@ class AddCommandSchema(Schema):
     snapshot_max_size: int = 0
     parser: str = "auto"
     plugins: str = ""
+    persona: str = "Default"
     only_new: bool | None = None
     update: bool = False
     overwrite: bool = False
@@ -121,14 +120,21 @@ def snapshot_filter_kwargs(args: SnapshotFilterCommandSchema, *, default_filter_
 @router.post("/add", response=CLICommandResponseSchema, summary="archivebox add [args] [urls]")
 def cli_add(request: HttpRequest, args: AddCommandSchema):
     from archivebox.cli.archivebox_add import add
+    from archivebox.misc.util import validate_url
 
     config_overrides: dict[str, object] = {}
     if args.only_new is not None:
         config_overrides["ONLY_NEW"] = bool(args.only_new)
     if args.update or args.overwrite:
         config_overrides["ONLY_NEW"] = False
+    submitted_urls: str | list[str] = args.urls
+    if len(args.urls) == 1:
+        try:
+            validate_url(args.urls[0])
+        except ValueError:
+            submitted_urls = args.urls[0]
     crawl, snapshots = add(
-        urls=args.urls,
+        urls=submitted_urls,
         snapshot_ids=args.snapshot_ids,
         tag=args.tag,
         depth=args.depth,
@@ -138,6 +144,7 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         snapshot_max_size=args.snapshot_max_size,
         index_only=args.index_only,
         plugins=args.plugins,
+        persona=args.persona,
         parser=args.parser,
         bg=True,  # Always run in background for API calls
         created_by_id=request.user.pk,

archivebox/api/v1_core.py

@@ -4,6 +4,7 @@ import math
 import json
 import mimetypes
 import re
+from html import unescape
 from collections import defaultdict
 from pathlib import Path, PurePosixPath
 from uuid import UUID
@@ -49,6 +50,7 @@ from archivebox.core.tag_util import (
     rename_tag as rename_tag_record,
 )
 from archivebox.crawls.models import Crawl
+from archivebox.crawls.locks import crawl_lifecycle_lock
 from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
 from archivebox.search.config import get_search_mode, get_search_mode_backend
 from archivebox.search.query import apply_snapshot_search
@@ -832,7 +834,7 @@ def _snapshots_rss_response(
         crawl_user = snapshot.crawl.created_by if snapshot.crawl_id else None
         description = f"Original URL: {snapshot.url}\nArchived snapshot: {archived_url}"
         feed.add_item(
-            title=snapshot.title or snapshot.url,
+            title=unescape(snapshot.title or snapshot.url),
             link=archived_url or web_base_url,
             description=description,
             unique_id=str(snapshot.id),
@@ -961,37 +963,38 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    snapshot_defaults = {
-        "depth": data.depth,
-        "title": data.title,
-        "timestamp": str(timezone.now().timestamp()),
-        "status": status or Snapshot.StatusChoices.QUEUED,
-        "retry_at": timezone.now(),
-    }
-    snapshot, _ = Snapshot.objects.get_or_create(
-        url=data.url,
-        crawl=crawl,
-        defaults=snapshot_defaults,
-    )
+    with crawl_lifecycle_lock(str(crawl.id)):
+        snapshot_defaults = {
+            "depth": data.depth,
+            "title": data.title,
+            "timestamp": str(timezone.now().timestamp()),
+            "status": status or Snapshot.StatusChoices.QUEUED,
+            "retry_at": timezone.now(),
+        }
+        snapshot, _ = Snapshot.objects.get_or_create(
+            url=data.url,
+            crawl=crawl,
+            defaults=snapshot_defaults,
+        )
 
-    update_fields: list[str] = []
-    if data.title is not None and snapshot.title != data.title:
-        snapshot.title = data.title
-        update_fields.append("title")
-    if status is not None and snapshot.status != status:
-        snapshot.status = status
-        update_fields.append("status")
-    if update_fields:
-        update_fields.append("modified_at")
-        snapshot.save(update_fields=update_fields)
+        update_fields: list[str] = []
+        if data.title is not None and snapshot.title != data.title:
+            snapshot.title = data.title
+            update_fields.append("title")
+        if status is not None and snapshot.status != status:
+            snapshot.status = status
+            update_fields.append("status")
+        if update_fields:
+            update_fields.append("modified_at")
+            snapshot.save(update_fields=update_fields)
 
-    if tags:
-        snapshot.save_tags(tags)
+        if tags:
+            snapshot.save_tags(tags)
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
+        try:
+            snapshot.ensure_crawl_symlink()
+        except Exception:
+            pass
 
     setattr(request, "with_archiveresults", False)
     return snapshot
@@ -1001,6 +1004,7 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
 def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateSchema):
     """Update a snapshot (e.g., set status=sealed to cancel queued work)."""
     snapshot = _get_snapshot_by_ref(snapshot_id)
+    crawl_id = str(snapshot.crawl_id)
 
     payload = data.dict(exclude_unset=True)
     update_fields = ["modified_at"]
@@ -1022,26 +1026,28 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
             return snapshot
         raise HttpError(400, f"Invalid action: {action}")
 
-    if "status" in payload:
-        try:
-            snapshot.status = normalize_snapshot_status(payload["status"])
-        except ValueError as err:
-            raise HttpError(400, str(err)) from err
-        if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
-            snapshot.retry_at = None
-        update_fields.append("status")
+    with crawl_lifecycle_lock(crawl_id):
+        snapshot = _get_snapshot_by_ref(snapshot_id)
+        if "status" in payload:
+            try:
+                snapshot.status = normalize_snapshot_status(payload["status"])
+            except ValueError as err:
+                raise HttpError(400, str(err)) from err
+            if snapshot.status == Snapshot.StatusChoices.SEALED and "retry_at" not in payload:
+                snapshot.retry_at = None
+            update_fields.append("status")
 
-    if "retry_at" in payload:
-        snapshot.retry_at = payload["retry_at"]
-        update_fields.append("retry_at")
+        if "retry_at" in payload:
+            snapshot.retry_at = payload["retry_at"]
+            update_fields.append("retry_at")
 
-    if tags is not None:
-        snapshot.save_tags(normalize_tag_list(tags))
+        if tags is not None:
+            snapshot.save_tags(normalize_tag_list(tags))
 
-    if payload.get("status") == Snapshot.StatusChoices.SEALED:
-        snapshot.cancel()
-    else:
-        snapshot.save(update_fields=update_fields)
+        if payload.get("status") == Snapshot.StatusChoices.SEALED:
+            snapshot.cancel()
+        else:
+            snapshot.save(update_fields=update_fields)
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1051,7 +1057,14 @@ def delete_snapshot(request: HttpRequest, snapshot_id: str):
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
     crawl_id_str = str(snapshot.crawl.pk)
-    deleted_count, _ = snapshot.delete()
+    snapshot.cancel()
+
+    from archivebox.services.runner import run_pending_crawls
+
+    with crawl_lifecycle_lock(crawl_id_str):
+        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
+        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
+        deleted_count, _ = snapshot.delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,

archivebox/api/v1_crawls.py

@@ -1,5 +1,6 @@
 __package__ = "archivebox.api"
 
+import json
 from pathlib import Path
 from uuid import UUID
 from datetime import datetime
@@ -48,8 +49,6 @@ class CrawlSchema(Schema):
     tags_str: str
     config: dict
 
-    # snapshots: List[SnapshotSchema]
-
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
@@ -124,7 +123,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
     config = dict(data.config or {})
     config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
-        urls="\n".join(urls),
+        urls="\n".join(json.dumps({"type": "CrawlSeed", "url": url, "depth": 0}, separators=(",", ":")) for url in urls),
         max_depth=data.max_depth,
         tags_str=",".join(tags),
         label=data.label,
@@ -250,8 +249,16 @@ def patch_crawl(request: HttpRequest, crawl_id: str, data: CrawlUpdateSchema):
 def delete_crawl(request: HttpRequest, crawl_id: str):
     crawl = get_crawl_by_ref(crawl_id)
     crawl_id_str = str(crawl.id)
-    snapshot_count = crawl.snapshot_set.count()
-    deleted_count, _ = crawl.delete()
+    crawl.cancel()
+
+    from archivebox.crawls.locks import crawl_lifecycle_lock
+    from archivebox.services.runner import run_pending_crawls
+
+    with crawl_lifecycle_lock(crawl_id_str):
+        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
+        crawl = get_crawl_by_ref(crawl_id_str)
+        snapshot_count = crawl.snapshot_set.count()
+        deleted_count, _ = crawl.delete()
     return {
         "success": True,
         "crawl_id": crawl_id_str,

archivebox/base_models/admin.py

@@ -5,11 +5,11 @@ __package__ = "archivebox.base_models"
 import json
 import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict, cast
+from typing import ClassVar, NotRequired, TypedDict, cast
 
 from django import forms
 from django.contrib import admin
-from django.db import models
+from django.db import DatabaseError, models
 from django.forms.renderers import BaseRenderer
 from django.http import HttpRequest, QueryDict
 from django.urls import path, register_converter
@@ -68,10 +68,10 @@ class KeyValueWidget(forms.Widget):
     template_name = ""  # We render manually
 
     class Media:
-        css = {
+        css: ClassVar[dict[str, list[str]]] = {
             "all": [],
         }
-        js = []
+        js: ClassVar[list[str]] = []
 
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
@@ -94,7 +94,7 @@ class KeyValueWidget(forms.Widget):
                             option[schema_key] = schema[schema_key]
                 options[key] = option
             return options
-        except Exception:
+        except (ImportError, KeyError, TypeError, ValueError):
             return {}
 
     def _parse_value(self, value: object) -> dict[str, object]:
@@ -644,41 +644,64 @@ class KeyValueWidget(forms.Widget):
 
                     window.updateHiddenField_{widget_id} = updateHiddenField_{widget_id};
 
-                    function focusConfigKeyFromHash_{widget_id}() {{
-                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
-                        // to (or creates) the matching row in this editor. Used by
-                        // the in-banner "pin via admin" link and the
-                        // ``→ Edit <KEY> in Machine.config`` shortcut on the live
-                        // config detail page.
-                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
-                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
-                            return;
-                        }}
+                    function configRowForKey_{widget_id}(key) {{
                         var container = document.getElementById('{widget_id}_rows');
                         if (!container) {{
-                            return;
+                            return null;
                         }}
                         var match = null;
                         container.querySelectorAll('.key-value-row').forEach(function(row) {{
                             if (match) {{ return; }}
                             var keyInput = row.querySelector('.kv-key');
-                            if (keyInput && keyInput.value.trim() === hash) {{
+                            if (keyInput && keyInput.value.trim() === key) {{
                                 match = row;
                             }}
                         }});
                         if (!match) {{
-                            // No existing row for this key — prepopulate one with the
-                            // key filled in but value left blank so the operator just
-                            // types/pastes the value and hits save.
                             window.addKeyValueRow_{widget_id}();
                             var rows = container.querySelectorAll('.key-value-row');
                             match = rows[rows.length - 1];
                             var keyInput = match.querySelector('.kv-key');
                             if (keyInput) {{
-                                keyInput.value = hash;
+                                keyInput.value = key;
                                 keyInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
                             }}
                         }}
+                        return match;
+                    }}
+
+                    function prefillConfigFromQuery_{widget_id}() {{
+                        var params = new URLSearchParams(window.location.search);
+                        var consumedConfigKeys = [];
+                        params.forEach(function(value, key) {{
+                            if (!/^[A-Z][A-Z0-9_]*$/.test(key) || !configMeta_{widget_id}[key]) {{
+                                return;
+                            }}
+                            consumedConfigKeys.push(key);
+                            var match = configRowForKey_{widget_id}(key);
+                            var valueInput = match && match.querySelector('.kv-value');
+                            if (valueInput) {{
+                                valueInput.value = value;
+                                valueInput.dispatchEvent(new Event('input', {{ bubbles: true }}));
+                            }}
+                        }});
+                        updateHiddenField_{widget_id}();
+                        if (consumedConfigKeys.length) {{
+                            var cleanUrl = new URL(window.location.href);
+                            consumedConfigKeys.forEach(function(key) {{ cleanUrl.searchParams.delete(key); }});
+                            window.history.replaceState(null, '', cleanUrl.pathname + cleanUrl.search + cleanUrl.hash);
+                        }}
+                    }}
+
+                    function focusConfigKeyFromHash_{widget_id}() {{
+                        // Deep-link affordance: ``…/change/#SOME_KEY`` jumps directly
+                        // to (or creates) the matching row in this editor. Used by
+                        // the setup wizard and live-config detail pages.
+                        var hash = (window.location.hash || '').replace(/^#/, '').trim();
+                        if (!hash || !/^[A-Z][A-Z0-9_]*$/.test(hash)) {{
+                            return;
+                        }}
+                        var match = configRowForKey_{widget_id}(hash);
                         if (!match) {{
                             return;
                         }}
@@ -700,12 +723,14 @@ class KeyValueWidget(forms.Widget):
                     // Initialize on load
                     document.addEventListener('DOMContentLoaded', function() {{
                         initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
                         focusConfigKeyFromHash_{widget_id}();
                     }});
                     // Also run immediately in case DOM is already ready
                     if (document.readyState !== 'loading') {{
                         initializeRows_{widget_id}();
+                        prefillConfigFromQuery_{widget_id}();
                         updateHiddenField_{widget_id}();
                         focusConfigKeyFromHash_{widget_id}();
                     }}
@@ -826,7 +851,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
         if change and obj.pk and obj.config is not None:
             try:
                 stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
-            except Exception:
+            except (AttributeError, DatabaseError, TypeError, ValueError):
                 stored = {}
             if isinstance(stored, dict):
                 new_config = dict(obj.config or {})
@@ -845,6 +870,20 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
     list_display = ("id", "created_at", "created_by")
     readonly_fields = ("id", "created_at", "modified_at")
     show_search_mode_selector = False
+    change_form_template = "admin/archivebox_change_form.html"
+
+    def get_admin_toolbar_actions(self, request, obj):
+        """Return extra action button dicts for the shared change-form toolbar.
+
+        See ``templates/admin/includes/archivebox_toolbar.html`` for the
+        accepted keys. Default: no extras (toolbar renders Save/History/Delete
+        plus any ``django-object-actions`` change_actions).
+        """
+        return []
+
+    def render_change_form(self, request, context, add=False, change=False, form_url="", obj=None):
+        context.setdefault("archivebox_admin_actions", self.get_admin_toolbar_actions(request, obj))
+        return super().render_change_form(request, context, add=add, change=change, form_url=form_url, obj=obj)
 
     def get_default_search_mode(self) -> str:
         # The shared changelist template always asks every admin for a default
@@ -877,9 +916,9 @@ class BaseModelAdmin(DjangoObjectActions, admin.ModelAdmin):
         """
         info = self.opts.app_label, self.opts.model_name
         object_routes = [
-            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="%s_%s_history" % info),
-            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="%s_%s_delete" % info),
-            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="%s_%s_change" % info),
+            path("<hexuuid:object_id>/history/", self.admin_site.admin_view(self.history_view), name="{}_{}_history".format(*info)),
+            path("<hexuuid:object_id>/delete/", self.admin_site.admin_view(self.delete_view), name="{}_{}_delete".format(*info)),
+            path("<hexuuid:object_id>/change/", self.admin_site.admin_view(self.change_view), name="{}_{}_change".format(*info)),
         ]
         # Append after super().get_urls() so our patterns are the
         # *last-registered* ones with the canonical admin URL names — Django's

archivebox/base_models/apps.py

@@ -1,7 +0,0 @@
-# from django.apps import AppConfig
-
-
-# class BaseModelsConfig(AppConfig):
-#     default_auto_field = 'django.db.models.BigAutoField'
-
-#     name = 'base_models'

archivebox/cli/__init__.py

@@ -18,10 +18,11 @@ if "--debug" in sys.argv:
 
 # Universal `--init` flag: when passed to ANY subcommand (e.g. `archivebox server --init`,
 # `archivebox add --init`, `archivebox shell --init`), run a `quick` archivebox init before
-# the subcommand executes. Strip it from argv here so each subcommand's own click parser
-# never sees it. Ignored for `help` and `init` themselves.
-if "--init" in sys.argv:
-    sys.argv = [arg for arg in sys.argv if arg != "--init"]
+# the subcommand executes. `--quick-init` is the old server spelling. Strip either from
+# argv here so each subcommand's own click parser never sees it. Ignored for `help` and
+# `init` themselves.
+if "--init" in sys.argv or "--quick-init" in sys.argv:
+    sys.argv = [arg for arg in sys.argv if arg not in ("--init", "--quick-init")]
     os.environ["ARCHIVEBOX_WANTS_INIT"] = "1"
 
 
@@ -32,6 +33,7 @@ class ArchiveBoxGroup(click.Group):
         "help": "archivebox.cli.archivebox_help.main",
         "version": "archivebox.cli.archivebox_version.main",
         "mcp": "archivebox.cli.archivebox_mcp.main",
+        "oneshot": "archivebox.cli.archivebox_oneshot.main",
     }
     setup_commands = {
         "init": "archivebox.cli.archivebox_init.main",
@@ -126,16 +128,12 @@ class ArchiveBoxGroup(click.Group):
             import_path = cmd_name_or_path
         modname, funcname = import_path.rsplit(".", 1)
 
-        # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
         func = vars(mod)[funcname]
 
         if func.__doc__ is None:
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
-        # if not isinstance(cmd, click.BaseCommand):
-        # raise ValueError(f'lazy loading of {import_path} failed - not a click command')
-
         return func
 
 
@@ -156,7 +154,6 @@ def cli(ctx, help=False):
     # then we need to set up the django environment and check that we're in a valid data folder
     wants_help = any(arg in ("-h", "--help", "--version") for arg in sys.argv[1:])
     if not wants_help and (subcommand in ArchiveBoxGroup.archive_commands or subcommand in ArchiveBoxGroup.model_commands):
-        # print('SETUP DJANGO AND CHECK DATA FOLDER')
         try:
             if subcommand == "server":
                 run_in_debug = "--reload" in sys.argv or os.environ.get("DEBUG") in ("1", "true", "True", "TRUE", "yes")

archivebox/cli/archivebox_add.py

@@ -5,14 +5,15 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox add"
 
 import sys
-import json
 import os
+import json
 from pathlib import Path
 
 from typing import Any, TYPE_CHECKING
 
 import rich_click as click
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.util import parse_filesize_to_bytes
 
@@ -23,21 +24,28 @@ if TYPE_CHECKING:
     from archivebox.core.models import Snapshot
 
 
-def _collect_input_urls(args: tuple[str, ...]) -> list[str]:
+def _collect_input_urls(args: tuple[str, ...], *, parser: str = "auto") -> list[str]:
     from archivebox.misc.jsonl import read_args_or_stdin
+    from archivebox.misc.util import validate_url
 
     urls: list[str] = []
     for record in read_args_or_stdin(args):
         url = record.get("url")
         if isinstance(url, str) and url:
-            urls.append(url)
+            try:
+                urls.append(validate_url(url))
+            except ValueError as err:
+                raise click.BadParameter(str(err), param_hint="URL") from err
 
         urls_field = record.get("urls")
         if isinstance(urls_field, str):
             for line in urls_field.splitlines():
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    urls.append(line)
+                    try:
+                        urls.append(validate_url(line))
+                    except ValueError as err:
+                        raise click.BadParameter(str(err), param_hint="URL") from err
 
     return urls
 
@@ -82,7 +90,6 @@ def add(
     crawl_max_size = parse_filesize_to_bytes(crawl_max_size)
     crawl_timeout = int(crawl_timeout or 0)
     snapshot_max_size = parse_filesize_to_bytes(snapshot_max_size)
-    from archivebox import CONSTANTS
     from archivebox.config.permissions import USER, HOSTNAME
     from archivebox.config.common import get_config
 
@@ -108,7 +115,6 @@ def add(
 
     # import models once django is set up
     from archivebox.crawls.models import Crawl
-    from archivebox.core.models import Snapshot
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.personas.models import Persona
     from archivebox.misc.logging_util import printable_filesize
@@ -119,36 +125,25 @@ def add(
     created_by_id = created_by_id or get_or_create_system_user_pk()
     started_at = timezone.now()
 
-    if isinstance(urls, str):
-        url_list = [line.strip() for line in urls.splitlines() if line.strip()]
-    else:
-        url_list = [str(url).strip() for url in urls if str(url).strip()]
-    if snapshot_ids and len(snapshot_ids) != len(url_list):
-        raise ValueError("snapshot_ids length must match urls length")
-    admitted_urls: list[str] = []
-    admitted_snapshot_ids: list[str] | None = [] if snapshot_ids else None
-    for index, url in enumerate(url_list):
-        if Snapshot.is_archivebox_internal_url(url):
-            print(f"[yellow][!] Skipping internal ArchiveBox URL: {url}[/yellow]")
-            continue
-        admitted_urls.append(url)
-        if admitted_snapshot_ids is not None and snapshot_ids:
-            admitted_snapshot_ids.append(snapshot_ids[index])
-
-    # 1. Save the provided URLs to sources/2024-11-05__23-59-59__cli_add.txt
-    sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__cli_add.txt"
-    sources_file.parent.mkdir(parents=True, exist_ok=True)
-    if admitted_snapshot_ids is not None:
-        sources_file.write_text(
-            "\n".join(
-                json.dumps({"url": url, "id": admitted_snapshot_ids[index], "tags": tag, "depth": 0})
-                for index, url in enumerate(admitted_urls)
-            ),
-        )
-    else:
-        sources_file.write_text("\n".join(admitted_urls))
+    use_internal_input_root = isinstance(urls, str)
+    source_text = (
+        urls
+        if use_internal_input_root
+        else "\n".join(json.dumps({"type": "CrawlSeed", "url": str(url), "depth": 0}, separators=(",", ":")) for url in urls)
+    )
 
     # 2. Create a new Crawl with inline URLs
+    # Foreground add must claim runner ownership before publishing runnable
+    # work. Otherwise an existing server runner can lease the new Crawl in the
+    # gap between Crawl.objects.create() and current_command(), then be killed
+    # by the newer add process halfway through its first Snapshot hook.
+    command = None
+    if not bg and not index_only:
+        from archivebox.machine.models import Process
+        from archivebox.core.takeover_util import current_command
+
+        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR)
+
     cli_args = [*sys.argv]
     if cli_args[0].lower().endswith("archivebox"):
         cli_args[0] = "archivebox"
@@ -156,10 +151,9 @@ def add(
 
     timestamp = timezone.now().strftime("%Y-%m-%d__%H-%M-%S")
 
-    urls_content = sources_file.read_text()
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
-    persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
+    persona_obj = Persona.get_or_create_named(persona_name)
     persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
@@ -186,20 +180,37 @@ def add(
     # stripped by Crawl.save() and rederived when hooks run.
     crawl_config.update(config_overrides)
 
-    crawl = Crawl.objects.create(
-        urls=urls_content,
-        max_depth=depth,
-        tags_str=tag,
-        persona_id=persona_obj.id,
-        label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
-        created_by_id=created_by_id,
-        status=Crawl.StatusChoices.QUEUED,
-        retry_at=None if index_only else timezone.now(),
-        config=crawl_config,
-    )
+    # Crawl.urls is the user's submitted source, not a derived work queue for
+    # this add path. Keeping it byte-for-byte readable is what lets API/UI/CLI
+    # callers audit or resume imports without losing RSS/Netscape/JSON metadata
+    # that is not representable as one plain URL per line.
+    try:
+        crawl = Crawl.objects.create(
+            urls=source_text,
+            # Stdin/import text gets an extra hop because the synthetic
+            # archivebox://internal root lives at depth 0 and parser-discovered
+            # URLs land at depth 1; direct URL args become the depth=0 input
+            # snapshots themselves so --depth=N matches the deepest hop the user
+            # asked for.
+            max_depth=depth + 1 if use_internal_input_root else depth,
+            tags_str=tag,
+            persona_id=persona_obj.id,
+            label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
+            created_by_id=created_by_id,
+            status=Crawl.StatusChoices.QUEUED,
+            retry_at=None if index_only else timezone.now(),
+            config=crawl_config,
+        )
+        first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
+    except BaseException:
+        if command is not None:
+            command.mark_exited(exit_code=1)
+        raise
+    if command is not None:
+        command.url = first_url
+        command.save(update_fields=["url", "modified_at"])
 
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
-    first_url = crawl.get_urls_list()[0] if crawl.get_urls_list() else ""
     print(f"    [dim]First URL: {first_url}[/dim]")
 
     # 3. The CrawlMachine will create Snapshots from all URLs when started
@@ -222,14 +233,16 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
+        from archivebox.services.runner import ensure_background_runner
+
+        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")
-        from archivebox.machine.models import Process
-        from archivebox.core.takeover_util import command_owns_foreground_runner, current_command, standby_until_foreground_runner_needed
+        from archivebox.core.takeover_util import command_owns_foreground_runner, standby_until_foreground_runner_needed
         from archivebox.workers.supervisord_util import get_existing_supervisord_process, run_runner_worker, stop_own_supervisord_process
 
-        command = current_command(Process.TypeChoices.ADD, data_dir=CONSTANTS.DATA_DIR, url=first_url)
+        assert command is not None
         exit_code = 0
         try:
             try:
@@ -240,6 +253,7 @@ def add(
                             ["--crawl-id", str(crawl.id)],
                             name=f"worker_runner_add_{os.getpid()}",
                             interactive_interrupts=True,
+                            config=get_config(crawl=crawl),
                         )
                         crawl.refresh_from_db(fields=["status", "retry_at"])
                         if exit_code == 0 and crawl.status == crawl.StatusChoices.SEALED:
@@ -336,6 +350,7 @@ def add(
 @click.option("--url-denylist", "--domain-denylist", default="", help="Comma-separated URL/domain denylist for this crawl")
 @click.option("--parser", default="auto", help="Parser for reading input URLs (auto, txt, html, rss, json, jsonl, netscape, ...)")
 @click.option("--plugins", "-p", default="", help="Comma-separated list of plugins to run e.g. title,favicon,screenshot,singlefile,...")
+@click.option("--extract", default="", help="Comma-separated plugins or output types to extract e.g. title,pdf,text/html,image,...")
 @click.option("--persona", default="Default", help="Authentication profile to use when archiving")
 @click.option(
     "--only-new/--no-only-new",
@@ -357,7 +372,12 @@ def main(**kwargs):
 
     with foreground_shutdown_signals(), foreground_parent_watchdog():
         raw_urls = kwargs.pop("urls")
-        urls = _collect_input_urls(raw_urls)
+        if raw_urls:
+            urls = _collect_input_urls(raw_urls, parser=kwargs.get("parser", "auto"))
+        elif not sys.stdin.isatty():
+            urls = sys.stdin.read()
+        else:
+            urls = []
         if not urls:
             raise click.UsageError("No URLs provided. Pass URLs as arguments or via stdin.")
         if int(kwargs.get("max_urls") or 0) < 0:
@@ -374,16 +394,33 @@ def main(**kwargs):
             raise click.BadParameter(str(err), param_hint="--snapshot-max-size") from err
         if kwargs.get("crawl_max_concurrent_snapshots") is not None and int(kwargs["crawl_max_concurrent_snapshots"]) < 1:
             raise click.BadParameter("crawl_max_concurrent_snapshots must be at least 1.", param_hint="--crawl-max-concurrent-snapshots")
+        if not kwargs.get("index_only"):
+            from archivebox.workers.supervisord_util import require_crawl_memory
+
+            require_crawl_memory()
 
         # Translate --only-new/--no-only-new into a crawl config override.
         # add() takes config overrides as a dict; no per-flag kwargs.
         overwrite = kwargs.pop("overwrite", False)
         update = kwargs.pop("update", False)
         only_new = kwargs.pop("only_new", None)
+        extract = kwargs.pop("extract", "")
         if overwrite or update:
             only_new = False
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
+        if extract:
+            from abx_dl.models import discover_plugins, plugins_matching_output
+
+            all_plugins = discover_plugins()
+            tokens = [token.strip() for token in extract.split(",") if token.strip()]
+            plugin_names = {name.lower(): name for name in all_plugins}
+            selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
+            selected += plugins_matching_output(all_plugins, tokens)
+            if not selected:
+                raise click.UsageError(f"No plugins found matching extract types: {extract}")
+            existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
+            kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
 
         add(urls=urls, **kwargs)
 

archivebox/cli/archivebox_config.py

@@ -59,28 +59,20 @@ def config(
     matching_config = {}
     if search:
         if config_options:
-            config_options = [
-                core_config_aliases.get(key.upper().strip()) or resolve_alias(key.upper().strip(), plugin_schemas) for key in config_options
-            ]
-            matching_config = {key: readable_config[key] for key in config_options if key in readable_config}
-            for config_section in CONFIGS.values():
-                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
+            search_terms = [key.strip().lower() for key in config_options]
 
-                for search_key in config_options:
-                    # search all aliases in the section
-                    for alias_key, key in aliases.items():
-                        if key in readable_config and search_key.lower() in alias_key.lower():
-                            matching_config[key] = dict(config_section)[key]
+            for existing_key, value in readable_config.items():
+                if any(term in existing_key.lower() or term in str(value).lower() for term in search_terms):
+                    matching_config[existing_key] = value
 
-                    # search all keys and values in the section
-                    for existing_key, value in dict(config_section).items():
-                        if existing_key in readable_config and (
-                            search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower()
-                        ):
-                            matching_config[existing_key] = value
-                    for existing_key, value in readonly_config.items():
-                        if search_key.lower() in existing_key.lower() or search_key.lower() in str(value).lower():
-                            matching_config[existing_key] = value
+            for alias, key in core_config_aliases.items():
+                if key in readable_config and any(term in alias.lower() for term in search_terms):
+                    matching_config[key] = readable_config[key]
+
+            for schema in plugin_schemas.values():
+                for key, metadata in schema.items():
+                    if key in readable_config and any(term in key.lower() or term in str(metadata).lower() for term in search_terms):
+                        matching_config[key] = readable_config[key]
 
         print(printable_config(matching_config))
         raise SystemExit(not matching_config)

archivebox/cli/archivebox_crawl.py

@@ -65,6 +65,7 @@ def create_crawl(
         1: Failure
     """
     from archivebox.misc.jsonl import read_args_or_stdin, write_record, TYPE_CRAWL
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
 
@@ -98,7 +99,11 @@ def create_crawl(
         # Collect URLs
         url = record.get("url")
         if url:
-            url_list.append(url)
+            try:
+                url_list.append(validate_url(str(url)))
+            except ValueError as err:
+                rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                return 1
 
         # Handle 'urls' field (newline-separated)
         urls_field = record.get("urls")
@@ -106,7 +111,11 @@ def create_crawl(
             for line in urls_field.split("\n"):
                 line = line.strip()
                 if line and not line.startswith("#"):
-                    url_list.append(line)
+                    try:
+                        url_list.append(validate_url(line))
+                    except ValueError as err:
+                        rprint(f"[red]Invalid URL: {err}[/red]", file=sys.stderr)
+                        return 1
 
     # Output pass-through records first
     if not is_tty:

archivebox/cli/archivebox_extract.py

@@ -46,7 +46,6 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     through the shared crawl runner with the corresponding plugin selected.
     """
     from rich import print as rprint
-    from django.utils import timezone
     from archivebox.core.models import ArchiveResult
     from archivebox.api.v1_core import _uuid_ref_query
     from archivebox.services.runner import run_crawl
@@ -60,22 +59,9 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
     rprint(f"[blue]Extracting {archiveresult.plugin} for {archiveresult.snapshot.url}[/blue]", file=sys.stderr)
 
     try:
-        was_paused = archiveresult.snapshot.is_paused
         archiveresult.reset_for_retry()
         snapshot = archiveresult.snapshot
-        if not was_paused:
-            snapshot.queue_for_extraction()
-        else:
-            # A paused snapshot may still accept explicit maintenance for one
-            # ArchiveResult, but this path must not transition it back to
-            # queued/startable work. Guard: only set retry_at while the row is
-            # still paused — concurrent resume would otherwise see a stale
-            # retry_at marker.
-            snapshot.safe_update(
-                {"retry_at": timezone.now()},
-                refresh=False,
-                extra_filter={"status": snapshot.StatusChoices.PAUSED},
-            )
+        snapshot.queue_for_extraction()
         crawl = snapshot.crawl
         if not crawl.claim_processing_lock(lock_seconds=10):
             rprint(
@@ -84,11 +70,7 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
             )
             return 1
 
-        try:
-            run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
-        finally:
-            if was_paused:
-                snapshot.restore_paused_scheduler_marker()
+        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -290,12 +272,9 @@ def run_plugins(
     queue_at = timezone.now()
     if existing_snapshot_ids:
         if requested_rows:
-            # Targeted ArchiveResult retries use retry_at as the scheduling
-            # signal and keep sealed snapshots sealed so extractors are not
-            # re-run outside the explicitly queued plugin rows. Paused snapshots
-            # also keep status=paused here: `retry_at` only asks the orchestrator
-            # to process the queued plugin rows, and run_due_snapshot restores
-            # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
+            # Search indexing on a sealed Snapshot is the only targeted hook
+            # allowed to bypass the normal lifecycle. Every other requested
+            # plugin requeues its Snapshot through the unified state machine.
             affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
             if preserve_queued and queued_rows:
                 queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
@@ -312,14 +291,30 @@ def run_plugins(
                         flat=True,
                     )
                 )
+            requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
+            for snapshot_id, plugin_name, _hook_name in requested_rows:
+                requested_plugins_by_id[snapshot_id].add(plugin_name)
             for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
-                snapshot.safe_update(
-                    {"retry_at": queue_at, "modified_at": queue_at},
-                    refresh=False,
-                    extra_filter={"status": snapshot.status},
+                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
+                sealed_search_backfill = (
+                    snapshot.status == Snapshot.StatusChoices.SEALED
+                    and plugin_names
+                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
                 )
+                if sealed_search_backfill:
+                    snapshot.safe_update(
+                        {"retry_at": queue_at, "modified_at": queue_at},
+                        refresh=False,
+                        extra_filter={"status": snapshot.status},
+                    )
+                else:
+                    snapshot.update_and_requeue(
+                        status=Snapshot.StatusChoices.QUEUED,
+                        retry_at=queue_at,
+                        current_step=0,
+                    )
         else:
             # No plugin rows were requested, so this is a full snapshot retry.
             for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "retry_at", "modified_at"):

archivebox/cli/archivebox_init.py

@@ -25,20 +25,20 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
+    from archivebox.misc.checks import check_not_inside_source_dir
+
+    check_not_inside_source_dir()
+
     from archivebox.config import CONSTANTS, VERSION
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
-    from archivebox.misc.db import apply_migrations
+    from archivebox.misc.db import apply_migrations, database_exists, ensure_database_ready
     from archivebox.misc.checks import check_migrations
 
     config = get_config()
 
-    # if os.access(out_dir / CONSTANTS.JSON_INDEX_FILENAME, os.F_OK):
-    #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
-    #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
-
     is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
-    existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
+    existing_index = database_exists()
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
         print("[green]----------------------------------------------------------------------[/green]")
@@ -86,11 +86,15 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
 
-    if os.access(CONSTANTS.DATABASE_FILE, os.F_OK):
+    if existing_index:
         print("\n[green][*] Verifying main SQL index and running any migrations needed...[/green]")
     else:
         print("\n[green][+] Building main SQL index and running initial migrations...[/green]")
 
+    # For postgres, make sure the server is reachable and create the database
+    # if it doesn't exist yet (sqlite creates its file automatically).
+    ensure_database_ready()
+
     from archivebox.config.django import setup_django
 
     setup_django()
@@ -107,12 +111,14 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
         else:
             os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
 
-    assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
-    print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
+    from archivebox.misc.db import database_display_location, is_postgres
 
-    # from django.contrib.auth.models import User
-    #     call_command("createsuperuser", interactive=True)
+    assert database_exists()
+    print()
+    if is_postgres():
+        print(f"    √ {database_display_location()}")
+    else:
+        print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
 
     print()
     print("[dodger_blue3][*] Checking links from indexes and archive folders (safe to Ctrl+C)...[/dodger_blue3]")
@@ -149,12 +155,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     CONSTANTS.PERSONAS_DIR.mkdir(parents=True, exist_ok=True)
     CONSTANTS.DEFAULT_TMP_DIR.mkdir(parents=True, exist_ok=True)
 
-    from archivebox.config.paths import get_or_create_working_tmp_dir, get_or_create_working_lib_dir
+    from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
 
     config = get_config()
-    config.TMP_DIR.mkdir(parents=True, exist_ok=True)
-    config.LIB_DIR.mkdir(parents=True, exist_ok=True)
-    (config.LIB_DIR / "bin").mkdir(parents=True, exist_ok=True)
+    create_and_chown_dir(config.TMP_DIR)
+    create_and_chown_dir(config.ABXPKG_LIB_DIR)
+    create_and_chown_dir(config.ABXPKG_LIB_DIR / "bin")
 
     working_tmp_dir = get_or_create_working_tmp_dir(autofix=True, quiet=True)
     if working_tmp_dir:
@@ -163,7 +169,6 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     working_lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True)
     if working_lib_dir:
         working_lib_dir.mkdir(parents=True, exist_ok=True)
-        (working_lib_dir / "bin").mkdir(parents=True, exist_ok=True)
 
     if install:
         from archivebox.cli.archivebox_install import install as install_method
@@ -187,7 +192,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
 @click.command()
 @click.option("--force", "-f", is_flag=True, help="Ignore unrecognized files in current directory and initialize anyway")
 @click.option("--quick", "-q", is_flag=True, help="Run any updates or migrations without rechecking all snapshot dirs")
-@click.option("--install", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
+@click.option("--install", "--setup", "-s", is_flag=True, help="Automatically install dependencies and extras used for archiving")
 @docstring(init.__doc__)
 def main(**kwargs) -> None:
     init(**kwargs)

archivebox/cli/archivebox_install.py

@@ -3,6 +3,9 @@
 __package__ = "archivebox.cli"
 
 import os
+import shutil
+import re
+from pathlib import Path
 
 import rich_click as click
 from rich import print
@@ -10,6 +13,108 @@ from rich import print
 from archivebox.misc.util import docstring, enforce_types
 
 
+def _resolve_install_targets(
+    requested_names: tuple[str, ...],
+) -> tuple[list[str], list[str]]:
+    """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
+    from abx_dl.models import discover_plugins, filter_plugins
+
+    plugins = discover_plugins(runtime="archivebox")
+    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in plugins}
+    plugin_names_by_binary_alias: dict[str, set[str]] = {}
+    for plugin_name, plugin in plugins.items():
+        for required_binary in plugin.config.required_binaries:
+            aliases = {required_binary.name.lower()}
+            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
+            if template_match:
+                config_key = template_match.group(1)
+                aliases.add(config_key.lower())
+                aliases.add(config_key.removesuffix("_BINARY").lower())
+                default = plugin.config.properties.get(config_key, {}).get("default")
+                if default:
+                    aliases.add(str(default).lower())
+            for alias in aliases:
+                plugin_names_by_binary_alias.setdefault(alias, set()).add(plugin_name)
+
+    requested_plugins: list[str] = []
+    raw_binary_names: list[str] = []
+    for name in requested_names:
+        normalized_name = name.lower()
+        if normalized_name in plugin_names_by_lower:
+            requested_plugins.append(plugin_names_by_lower[normalized_name])
+        elif normalized_name in plugin_names_by_binary_alias:
+            requested_plugins.extend(sorted(plugin_names_by_binary_alias[normalized_name]))
+        else:
+            raw_binary_names.append(name)
+
+    selected_plugins = filter_plugins(plugins, requested_plugins, include_providers=True) if requested_plugins else {}
+    selected_plugin_names = {name.lower() for name in selected_plugins}
+    raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
+    return sorted(selected_plugins), sorted(set(raw_binary_names))
+
+
+def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
+    """Install user-requested standalone binaries through the Binary state machine."""
+    from django.utils import timezone
+
+    from archivebox.machine.models import Binary, Machine, _canonical_binary_name
+    from archivebox.services.runner import run_due_binary
+
+    machine = Machine.current()
+    for requested_name in binary_names:
+        binary_name = _canonical_binary_name(requested_name)
+        if not binary_name:
+            raise click.UsageError(f"Invalid binary name: {requested_name}")
+        binary = Binary.objects.filter(machine=machine, name=binary_name).order_by("-modified_at").first()
+        if binary is None:
+            binary = Binary.objects.create(
+                machine=machine,
+                name=binary_name,
+                binproviders=binproviders,
+                overrides={},
+                status=Binary.StatusChoices.QUEUED,
+            )
+        elif not binary.is_valid or binary.binproviders != binproviders:
+            binary.binproviders = binproviders
+            binary.overrides = {}
+            binary.status = Binary.StatusChoices.QUEUED
+            binary.retry_at = timezone.now()
+            binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
+        run_due_binary(binary, lock_seconds=60)
+
+
+def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
+    """Expose the shared abxpkg lib dir at DATA_DIR/lib after install."""
+    link_path = data_dir / "lib"
+    target_config_path = abxpkg_lib_dir.expanduser().absolute()
+    target_path = abxpkg_lib_dir.expanduser().resolve()
+    from archivebox.config.paths import get_machine_type
+
+    scoped_data_lib_path = (link_path / get_machine_type()).absolute()
+
+    if target_config_path == scoped_data_lib_path:
+        if link_path.is_symlink() or link_path.is_file():
+            link_path.unlink()
+        target_config_path.mkdir(parents=True, exist_ok=True)
+        return None
+
+    if link_path.resolve(strict=False) == target_path:
+        return None
+
+    target_path.mkdir(parents=True, exist_ok=True)
+
+    if link_path.is_symlink():
+        if link_path.resolve(strict=False) == target_path:
+            return link_path
+        link_path.unlink()
+    elif link_path.exists():
+        shutil.rmtree(link_path)
+
+    relative_target = os.path.relpath(target_path, start=link_path.parent)
+    link_path.symlink_to(relative_target, target_is_directory=True)
+    return link_path
+
+
 @enforce_types
 def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bool = False) -> None:
     """Detect and install ArchiveBox dependencies by running the abx-dl install flow
@@ -56,16 +161,32 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
 
     setup_django()
 
-    plugin_names = list(binaries)
+    plugin_names, raw_binary_names = _resolve_install_targets(binaries) if binaries else ([], [])
+    install_plugin_names = list(plugin_names)
     if binproviders != "*":
-        plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
+        install_plugin_names.extend(provider.strip() for provider in binproviders.split(",") if provider.strip())
 
-    print("[+] Running installer via abx-dl bus...")
-    print()
+    if install_plugin_names or not binaries:
+        if not binaries:
+            from archivebox.workers.supervisord_util import require_dependency_install_memory
 
-    from archivebox.services.runner import run_install
+            require_dependency_install_memory()
+        print("[+] Running plugin installer via abx-dl bus...")
+        print()
 
-    run_install(plugin_names=plugin_names or None)
+        from archivebox.services.runner import run_install
+
+        run_install(plugin_names=install_plugin_names or None)
+
+    if raw_binary_names:
+        print(f"[+] Running direct binary installer via ArchiveBox binary state machine: {', '.join(raw_binary_names)}")
+        print()
+        _install_raw_binary_names(raw_binary_names, binproviders)
+
+    from archivebox.config.common import get_config
+
+    config = get_config(include_machine=False)
+    ensure_data_dir_lib_symlink(CONSTANTS.DATA_DIR, config.ABXPKG_LIB_DIR)
 
     print()
 
@@ -84,7 +205,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     # Django is already loaded, so just import and call the function directly
     from archivebox.cli.archivebox_version import version as show_version
 
-    show_version(quiet=False)
+    show_version(quiet=False, binaries=binaries)
 
 
 @click.command()

archivebox/cli/archivebox_manage.py

@@ -14,7 +14,7 @@ def manage(args: list[str] | None = None) -> None:
     from archivebox.misc.logging import stderr
 
     config = get_config()
-    if (args and "createsuperuser" in args) and (config.IN_DOCKER and not config.IS_TTY):
+    if (args and "createsuperuser" in args and "--noinput" not in args) and (config.IN_DOCKER and not config.IS_TTY):
         stderr("[!] Warning: you need to pass -it to use interactive commands in docker", color="lightyellow")
         stderr("    docker run -it archivebox manage {}".format(" ".join(args or ["..."])), color="lightyellow")
         stderr("")

archivebox/cli/archivebox_oneshot.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+__package__ = "archivebox.cli"
+__command__ = "archivebox oneshot"
+
+import subprocess
+import sys
+from pathlib import Path
+
+import rich_click as click
+
+from archivebox.config import CONSTANTS
+from archivebox.config.common import get_config
+
+
+@click.command(add_help_option=False, context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
+def main(args: tuple[str, ...] = ()) -> None:
+    """Download URLs using abx-dl"""
+    from archivebox.misc.checks import check_not_inside_source_dir
+
+    check_not_inside_source_dir()
+
+    cwd = Path.cwd()
+    if any((path / CONSTANTS.SQL_INDEX_FILENAME).exists() for path in (cwd, *cwd.parents)):
+        raise click.ClickException(
+            "Refusing to run `archivebox oneshot` inside an ArchiveBox DATA_DIR. Use `archivebox add` here, or run oneshot from another directory.",
+        )
+    abxpkg_binary = Path(sys.executable).with_name("abxpkg")
+    if not abxpkg_binary.is_file():
+        raise click.ClickException(f"abxpkg executable is missing from the ArchiveBox environment: {abxpkg_binary}")
+    abxpkg_lib_dir = get_config(include_machine=False).ABXPKG_LIB_DIR
+    raise SystemExit(
+        subprocess.run(
+            [
+                str(abxpkg_binary),
+                f"--lib={abxpkg_lib_dir}",
+                "--binproviders=env",
+                "--install",
+                "run",
+                "abx-dl",
+                *args,
+            ],
+        ).returncode,
+    )
+
+
+if __name__ == "__main__":
+    main()

archivebox/cli/archivebox_persona.py

@@ -137,65 +137,6 @@ def get_edge_user_data_dir() -> Path | None:
     return None
 
 
-def get_browser_binary(browser: str) -> str | None:
-    system = platform.system()
-    home = Path.home()
-    browser = browser.lower()
-
-    if system == "Darwin":
-        candidates = {
-            "chrome": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
-            "chromium": ["/Applications/Chromium.app/Contents/MacOS/Chromium"],
-            "brave": ["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"],
-            "edge": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
-        }.get(browser, [])
-    elif system == "Linux":
-        candidates = {
-            "chrome": [
-                "/usr/bin/google-chrome",
-                "/usr/bin/google-chrome-stable",
-                "/usr/bin/google-chrome-beta",
-                "/usr/bin/google-chrome-unstable",
-            ],
-            "chromium": ["/usr/bin/chromium", "/usr/bin/chromium-browser"],
-            "brave": ["/usr/bin/brave-browser", "/usr/bin/brave-browser-beta", "/usr/bin/brave-browser-nightly"],
-            "edge": [
-                "/usr/bin/microsoft-edge",
-                "/usr/bin/microsoft-edge-stable",
-                "/usr/bin/microsoft-edge-beta",
-                "/usr/bin/microsoft-edge-dev",
-            ],
-        }.get(browser, [])
-    elif system == "Windows":
-        local_app_data = Path(os.environ.get("LOCALAPPDATA", home / "AppData" / "Local"))
-        candidates = {
-            "chrome": [
-                str(local_app_data / "Google" / "Chrome" / "Application" / "chrome.exe"),
-                "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
-                "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
-            ],
-            "chromium": [str(local_app_data / "Chromium" / "Application" / "chrome.exe")],
-            "brave": [
-                str(local_app_data / "BraveSoftware" / "Brave-Browser" / "Application" / "brave.exe"),
-                "C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-                "C:\\Program Files (x86)\\BraveSoftware\\Brave-Browser\\Application\\brave.exe",
-            ],
-            "edge": [
-                str(local_app_data / "Microsoft" / "Edge" / "Application" / "msedge.exe"),
-                "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
-                "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
-            ],
-        }.get(browser, [])
-    else:
-        candidates = []
-
-    for candidate in candidates:
-        if candidate and Path(candidate).exists():
-            return candidate
-
-    return None
-
-
 BROWSER_PROFILE_FINDERS = {
     "chrome": get_chrome_user_data_dir,
     "chromium": get_chrome_user_data_dir,  # Same locations
@@ -317,10 +258,6 @@ def create_personas(
         if profile is None and (source_profile_dir / "Default").exists():
             profile = "Default"
 
-        browser_binary = get_browser_binary(import_from)
-        if browser_binary:
-            rprint(f"[dim]Using {import_from} binary: {browser_binary}[/dim]", file=sys.stderr)
-
     created_count = 0
     for name in name_list:
         name = name.strip()

archivebox/cli/archivebox_remove.py

@@ -9,7 +9,6 @@ from collections.abc import Iterable
 
 import rich_click as click
 
-from django.db import OperationalError
 from django.db.models import QuerySet
 
 from archivebox.config import CONSTANTS
@@ -34,6 +33,7 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
+    delete: bool = False,
     out_dir: Path = CONSTANTS.DATA_DIR,
     timeout: float | None = None,
     **kwargs,
@@ -91,12 +91,6 @@ def remove(
     # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
     # rmtree doesn't hold the lock either.
     #
-    # The SQLite retry wrapper in core/sqlite_backend/base.py re-raises lock
-    # errors when called inside an atomic block (because it can't safely
-    # release+reacquire a transaction), so we wrap each row's delete in our
-    # own retry loop at this outer (non-atomic) level. Each attempt is a
-    # fresh atomic; an exception cleanly rolls it back before we sleep.
-    retry_interval = 1.0
     deleted_snapshot_pks = []
     timed_out = False
     timeout_error = ""
@@ -105,23 +99,9 @@ def remove(
             timed_out = True
             timeout_error = f"Remove timed out after {timeout:g}s with {len(snapshot_pks) - index} snapshots remaining."
             break
-        while True:
-            try:
-                deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
-                if deleted_count:
-                    deleted_snapshot_pks.append(pk)
-                break
-            except OperationalError as err:
-                if "database is locked" not in str(err):
-                    raise
-                remaining_time = deadline - time.monotonic() if deadline is not None else None
-                if remaining_time is not None and remaining_time <= 0:
-                    timed_out = True
-                    timeout_error = f"Remove timed out after {timeout:g}s while waiting for the database lock."
-                    break
-                time.sleep(min(retry_interval, remaining_time) if remaining_time is not None else retry_interval)
-        if timed_out:
-            break
+        deleted_count, _ = Snapshot.objects.filter(pk=pk).delete()
+        if deleted_count:
+            deleted_snapshot_pks.append(pk)
 
     all_snapshots = Snapshot.objects.all()
     remaining_count = all_snapshots.count()
@@ -142,6 +122,7 @@ def remove(
 
 @click.command()
 @click.option("--yes", is_flag=True, help="Remove links instantly without prompting to confirm")
+@click.option("--delete", is_flag=True, help="Compatibility no-op; archived content is deleted automatically")
 @click.option("--timeout", type=float, default=None, help="Maximum seconds to spend deleting snapshots")
 @snapshot_filter_options(default_filter_type="exact")
 @docstring(remove.__doc__)

archivebox/cli/archivebox_run.py

@@ -248,7 +248,12 @@ def process_stdin_records() -> int:
     return 0
 
 
-def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False) -> int:
+def run_runner(
+    daemon: bool = False,
+    crawl_id: str | None = None,
+    maintenance_only: bool = False,
+    maintenance_batch_size: int | None = None,
+) -> int:
     """
     Run the background runner loop.
 
@@ -282,24 +287,18 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
         current.mark_exited()
         return 0
 
-    recover_orchestrator_state(include_chrome=True)
+    recover_orchestrator_state(include_chrome=crawl_id is None, crawl_id=crawl_id)
     if crawl_id:
         from django.utils import timezone
         from archivebox.crawls.models import Crawl
 
         crawl = Crawl.objects.filter(id=crawl_id, status__in=Crawl.RUNNABLE_STATES).first()
         now = timezone.now()
-        # Only re-lease when the row is unscheduled (retry_at IS NULL) or its
-        # existing lease has already expired. A future retry_at means another
-        # worker is already scheduled — don't clobber.
-        if crawl is not None and (crawl.retry_at is None or crawl.retry_at <= now):
-            # extra_filter pins the read-time retry_at so a concurrent worker
-            # that grabbed the lease between our SELECT and UPDATE wins.
-            crawl.safe_update(
-                {"retry_at": now},
-                refresh=False,
-                extra_filter={"retry_at": crawl.retry_at},
-            )
+        # Winning the single-runner gate terminates and waits for every older
+        # runner. Requeue the explicitly requested crawl so an abandoned
+        # future ownership lease cannot hide it from the unified scheduler.
+        if crawl is not None:
+            crawl.update_and_requeue(retry_at=now, refresh=False)
     # Only a foreground `archivebox add` gets the interactive "abort current
     # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
     # orchestrators should shut down immediately and cleanly on the first signal.
@@ -319,6 +318,7 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
                 crawl_id=crawl_id,
                 maintenance_only=maintenance_only,
                 interactive_interrupts=interactive_interrupts,
+                **({"maintenance_batch_size": maintenance_batch_size} if maintenance_batch_size else {}),
             )
         return 0
     except KeyboardInterrupt:
@@ -342,9 +342,24 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
 @click.option("--crawl-id", help="Run the crawl runner for a specific crawl only")
 @click.option("--snapshot-id", help="Run one snapshot through its crawl")
 @click.option("--binary-id", help="Run one queued binary install directly on the bus")
-@click.option("--maintenance-only", is_flag=True, help="Only process due maintenance ticks on sealed/paused snapshots")
+@click.option("--maintenance-only", is_flag=True, help="Only process sealed Snapshot maintenance and search-index backfills")
+@click.option(
+    "--maintenance-batch-size",
+    type=int,
+    default=None,
+    hidden=True,
+    help="Limit queued maintenance snapshots claimed per scheduler tick",
+)
 @click.option("--no-stdin", is_flag=True, hidden=True, help="Run the scheduler even when stdin is not a TTY")
-def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintenance_only: bool, no_stdin: bool):
+def main(
+    daemon: bool,
+    crawl_id: str,
+    snapshot_id: str,
+    binary_id: str,
+    maintenance_only: bool,
+    maintenance_batch_size: int | None,
+    no_stdin: bool,
+):
     """
     Process queued work.
 
@@ -367,7 +382,7 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 ),
                 foreground_parent_watchdog(enabled=False),
             ):
-                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
+                sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
         except KeyboardInterrupt:
             sys.exit(0)
 
@@ -391,23 +406,38 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
                 sys.exit(1)
 
         if crawl_id:
-            sys.exit(run_runner(daemon=False, crawl_id=crawl_id, maintenance_only=maintenance_only))
+            sys.exit(
+                run_runner(
+                    daemon=False,
+                    crawl_id=crawl_id,
+                    maintenance_only=maintenance_only,
+                    maintenance_batch_size=maintenance_batch_size,
+                ),
+            )
 
         if maintenance_only:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=True))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=True, maintenance_batch_size=maintenance_batch_size))
 
         if not no_stdin and not sys.stdin.isatty():
             sys.exit(process_stdin_records())
         else:
-            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only))
+            sys.exit(run_runner(daemon=daemon, maintenance_only=maintenance_only, maintenance_batch_size=maintenance_batch_size))
 
 
 def run_snapshot_worker(snapshot_id: str) -> int:
+    from archivebox.config import CONSTANTS
+    from archivebox.core.takeover_util import enter_single_runner_gate
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
+    from archivebox.machine.models import Process
     from archivebox.core.models import Snapshot
     from archivebox.services.runner import run_due_snapshot
     from django.utils import timezone
 
+    current = Process.current()
+    if not enter_single_runner_gate(current, data_dir=CONSTANTS.DATA_DIR):
+        current.mark_exited()
+        return 0
+
     snapshot = None
     try:
         with foreground_shutdown_signals(), foreground_parent_watchdog():
@@ -437,6 +467,10 @@ def run_snapshot_worker(snapshot_id: str) -> int:
 
         traceback.print_exc()
         return 1
+    finally:
+        current.refresh_from_db()
+        if current.status != Process.StatusChoices.EXITED:
+            current.mark_exited()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_server.py

@@ -231,6 +231,7 @@ def server(
     debug: bool = False,
     daemonize: bool = False,
     nothreading: bool = False,
+    createsuperuser: bool = False,
 ) -> None:
     """Run the ArchiveBox HTTP server"""
     from archivebox.config.common import get_config
@@ -244,6 +245,12 @@ def server(
 
     from django.contrib.auth.models import User
 
+    if createsuperuser:
+        from archivebox.cli.archivebox_manage import manage
+
+        manage(args=["createsuperuser"])
+        print()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         print()
         print(
@@ -260,11 +267,13 @@ def server(
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
     if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
+        from archivebox.workers.supervisord_util import resolve_env_binary
+
         log_path = CONSTANTS.LOGS_DIR / "server.log"
         log_path.parent.mkdir(parents=True, exist_ok=True)
         daemon_env = os.environ.copy()
         daemon_env["ARCHIVEBOX_SERVER_DAEMON_CHILD"] = "1"
-        daemon_cmd = [sys.executable, "-m", "archivebox", "server"]
+        daemon_cmd = [str(resolve_env_binary("archivebox")), "server"]
         if debug:
             daemon_cmd.append("--debug")
         if reload:
@@ -293,7 +302,14 @@ def server(
             except OSError:
                 time.sleep(0.1)
         else:
-            print(f"[yellow][!] ArchiveBox daemon server pid={proc.pid} is still starting. See {log_path}[/yellow]")
+            print(f"[red][X] ArchiveBox daemon server pid={proc.pid} did not become ready. See {log_path}[/red]")
+            proc.terminate()
+            try:
+                proc.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                proc.wait(timeout=5)
+            sys.exit(1)
         return
 
     os.environ["BIND_ADDR"] = f"{host}:{port}"
@@ -359,7 +375,7 @@ def server(
                     data_dir=CONSTANTS.DATA_DIR,
                     exclude_id=command.id,
                 )
-                takeover_components = active_supervisord_runtime_components(config=config)
+                takeover_components = active_supervisord_runtime_components()
                 if older_owner and takeover_components:
                     print(
                         "[yellow][*] Taking over "
@@ -404,6 +420,7 @@ def server(
 @click.option("--debug", is_flag=True, help="Enable DEBUG=True mode with more verbose errors")
 @click.option("--nothreading", is_flag=True, help="Force runserver to run in single-threaded mode")
 @click.option("--daemonize", is_flag=True, help="Run the server in the background as a daemon")
+@click.option("--createsuperuser", is_flag=True, help="Run archivebox manage createsuperuser before starting the server")
 @docstring(server.__doc__)
 def main(**kwargs):
     server(**kwargs)

archivebox/cli/archivebox_snapshot.py

@@ -31,14 +31,62 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox snapshot"
 
 import sys
-from collections.abc import Iterable
+from collections.abc import Iterable, Iterator
+from itertools import islice
 
 import rich_click as click
 from rich import print as rprint
 from django.db.models import QuerySet
 
 SNAPSHOT_FILTER_TYPE_CHOICES = ("exact", "substring", "regex", "domain", "tag", "timestamp")
-SNAPSHOT_LIST_CHUNK_SIZE = 100
+SNAPSHOT_LIST_CHUNK_SIZE = 5000
+
+
+def iter_snapshot_json(queryset: QuerySet) -> Iterator[dict[str, object]]:
+    from archivebox.config import VERSION
+    from archivebox.core.models import SnapshotTag
+
+    fields = (
+        "id",
+        "crawl_id",
+        "url",
+        "title",
+        "bookmarked_at",
+        "created_at",
+        "timestamp",
+        "depth",
+        "status",
+        "fs_version",
+        "output_size",
+    )
+    rows = queryset.values(*fields).iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE)
+    while batch := list(islice(rows, SNAPSHOT_LIST_CHUNK_SIZE)):
+        tags_by_snapshot = {row["id"]: [] for row in batch}
+        tag_rows = (
+            SnapshotTag.objects.filter(snapshot_id__in=tags_by_snapshot).order_by("tag__name").values_list("snapshot_id", "tag__name")
+        )
+        for snapshot_id, tag_name in tag_rows:
+            tags_by_snapshot[snapshot_id].append(tag_name)
+
+        for row in batch:
+            archive_size = int(row["output_size"] or 0)
+            yield {
+                "type": "Snapshot",
+                "schema_version": VERSION,
+                "id": str(row["id"]),
+                "crawl_id": str(row["crawl_id"]),
+                "url": row["url"],
+                "title": row["title"],
+                "tags": ",".join(sorted(tags_by_snapshot[row["id"]])),
+                "bookmarked_at": row["bookmarked_at"].isoformat() if row["bookmarked_at"] else None,
+                "created_at": row["created_at"].isoformat() if row["created_at"] else None,
+                "timestamp": row["timestamp"],
+                "depth": row["depth"],
+                "status": row["status"],
+                "fs_version": row["fs_version"],
+                "archive_size": archive_size,
+                "output_size": archive_size,
+            }
 
 
 # =============================================================================
@@ -67,7 +115,7 @@ def create_snapshots(
         TYPE_SNAPSHOT,
         TYPE_CRAWL,
     )
-    from archivebox.misc.util import validate_url_length
+    from archivebox.misc.util import validate_url
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
@@ -112,7 +160,7 @@ def create_snapshots(
                 # Create snapshots for each URL in the crawl
                 for url in crawl.get_urls_list():
                     try:
-                        validate_url_length(url)
+                        validate_url(url)
                     except ValueError as err:
                         rprint(f"[red]Error creating snapshot: {err}[/red]", file=sys.stderr)
                         continue
@@ -135,7 +183,7 @@ def create_snapshots(
             elif record_type == TYPE_SNAPSHOT or record.get("url"):
                 # Input is a Snapshot or plain URL
                 if record.get("url"):
-                    validate_url_length(str(record["url"]))
+                    validate_url(str(record["url"]))
                 if tag and not record.get("tags"):
                     record["tags"] = tag
                 if status:
@@ -296,8 +344,8 @@ def list_snapshots(
         return 0
 
     if not is_tty:
-        for snapshot in queryset.prefetch_related("tags").iterator(chunk_size=SNAPSHOT_LIST_CHUNK_SIZE):
-            write_record(snapshot.to_json())
+        for snapshot_json in iter_snapshot_json(queryset):
+            write_record(snapshot_json)
             count += 1
         rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
         return 0

archivebox/cli/archivebox_status.py

@@ -44,12 +44,18 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
         legacy_snapshot_dirs = [
             entry for entry in archive_dir.iterdir() if entry.is_dir() and not entry.is_symlink() and Snapshot.is_legacy_archive_dir(entry)
         ]
-    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {CONSTANTS.SQL_INDEX_FILENAME})")
+    from archivebox.misc.db import database_display_location, is_postgres
+
+    index_location = database_display_location() if is_postgres() else CONSTANTS.SQL_INDEX_FILENAME
+    print(f"    > SQL Main Index: {num_sql_links} links".ljust(36), f"(found in {index_location})")
     print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
     users_dir = CONSTANTS.USERS_DIR
-    scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
+    # The current users layout is nested below archive/, so scanning both
+    # archive/ and archive/users/ recursively counts every current snapshot
+    # twice. A single archive root includes both legacy and current layouts.
+    scan_roots = [archive_dir] if archive_dir.exists() else []
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")
     do_precise_fs_scan = num_sql_links <= MAX_STATUS_FS_DIR_SCAN
@@ -109,7 +115,7 @@ def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
 
     if orphaned_dirs:
         print("    [violet]Hint:[/violet] To automatically import orphaned data directories into the main index, run:")
-        print("        [green]archivebox init[/green]")
+        print("        [green]archivebox update[/green]")
 
     print()
     print("[green]\\[*] Scanning recent archive changes and user logins:[/green]")

archivebox/cli/archivebox_update.py

@@ -76,9 +76,14 @@ def reindex_snapshots(
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
-    from archivebox.core.models import ArchiveResult
+    from archivebox.core.models import ArchiveResult, Snapshot
     from abx_dl.models import discover_plugins
 
+    # Search backfill is the one maintenance hook allowed to execute without
+    # reopening a Snapshot. Restrict that exception to already-sealed rows;
+    # every open lifecycle state remains owned by the normal state machine.
+    snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
+
     stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
     plugins_by_name = discover_plugins(runtime="archivebox")
@@ -227,24 +232,34 @@ def update(
     from archivebox.core.takeover_util import (
         command_owns_foreground_runner,
         current_command,
-        ensure_daemon_stack,
+        foreground_runner_owner,
         standby_until_foreground_runner_needed,
     )
     from archivebox.workers.supervisord_util import run_runner_worker, stop_own_supervisord_process
 
     command = current_command(Process.TypeChoices.UPDATE, data_dir=CONSTANTS.DATA_DIR)
 
+    def still_owns_foreground_runner() -> bool:
+        from django.db import connections
+
+        try:
+            return command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR)
+        finally:
+            connections.close_all()
+
     def wait_for_turn() -> None:
         raise_if_shutdown_requested()
         standby_until_foreground_runner_needed(command, data_dir=CONSTANTS.DATA_DIR)
         raise_if_shutdown_requested()
 
-    def run_scoped_runner(*args: str, ensure_daemon_reason: str | None = None) -> None:
+    def run_scoped_runner(*args: str) -> None:
         while True:
             wait_for_turn()
-            if ensure_daemon_reason:
-                ensure_daemon_stack(reason=ensure_daemon_reason)
-            exit_code = run_runner_worker(list(args), name=f"worker_runner_update_{os.getpid()}")
+            exit_code = run_runner_worker(
+                list(args),
+                name=f"worker_runner_update_{os.getpid()}",
+                keep_running=still_owns_foreground_runner,
+            )
             if exit_code == 0:
                 return
             if not command_owns_foreground_runner(command, data_dir=CONSTANTS.DATA_DIR):
@@ -352,7 +367,7 @@ def update(
                         # Snapshot rows: Snapshot.save() moves archive/<ts> to
                         # the current output_dir and preserves the lifecycle
                         # status. Drain those retry_at ticks before queuing
-                        # search backfill below. Otherwise the sealed/paused
+                        # search backfill below. Otherwise the sealed search
                         # runner branch correctly sees queued ArchiveResult
                         # rows first, runs the targeted plugins, and may leave
                         # the fs_version maintenance tick hidden behind that
@@ -373,14 +388,13 @@ def update(
                             for snapshot_id in sorted(touched_snapshot_ids):
                                 run_scoped_runner("--snapshot-id", snapshot_id)
                         else:
-                            run_scoped_runner("--maintenance-only")
+                            run_scoped_runner("--maintenance-only", "--maintenance-batch-size", str(batch_size))
                         ran_post_migrate_runner = True
 
                 if do_index:
                     if full_update_empty:
                         print("[*] No snapshots found; skipping search indexing backfill.")
                     else:
-                        ensure_daemon_stack(reason="search indexing")
                         search_plugins = _get_search_indexing_plugins()
                         if not search_plugins:
                             print("[*] No search indexing plugins are available, nothing to backfill.")
@@ -417,9 +431,7 @@ def update(
                                 now = timezone.now()
                                 queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
                                 snapshots_to_wake = (
-                                    Snapshot.objects.filter(
-                                        status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
-                                    )
+                                    Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED)
                                     .annotate(
                                         has_queued_index_result=Exists(queued_result_for_snapshot),
                                     )
@@ -441,11 +453,19 @@ def update(
                                     f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
                                 )
                             else:
+                                collect_index_ids = (
+                                    is_filtered_update
+                                    or foreground_runner_owner(
+                                        data_dir=CONSTANTS.DATA_DIR,
+                                        exclude_id=command.id,
+                                    )
+                                    is not None
+                                )
                                 stats = reindex_snapshots(
                                     snapshots,
                                     search_plugins=search_plugins,
                                     batch_size=batch_size,
-                                    collect_ids=True,
+                                    collect_ids=collect_index_ids,
                                     wait_for_turn=wait_for_turn,
                                 )
                                 print_index_stats(stats)
@@ -454,8 +474,8 @@ def update(
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
-                    # ArchiveResult rows without reopening sealed/paused
-                    # snapshots. This second runner pass drains those plugin
+                    # ArchiveResult rows without reopening sealed snapshots.
+                    # This second runner pass drains those plugin
                     # rows after filesystem maintenance has had its own turn.
                     # For a normal unfiltered `archivebox update`, keep the
                     # historical final pass broad enough to resume genuinely
@@ -470,15 +490,14 @@ def update(
                         pass
                     elif not runner_work_queued:
                         pass
-                    elif touched_snapshot_ids:
+                    elif touched_snapshot_ids and is_filtered_update:
                         if not touched_snapshot_ids:
                             print("[*] No matching snapshots queued work for the runner.")
                         for snapshot_id in sorted(touched_snapshot_ids):
                             run_scoped_runner("--snapshot-id", snapshot_id)
                     else:
                         run_scoped_runner(
-                            *(["--maintenance-only"] if index_only or migrate_only else []),
-                            ensure_daemon_reason="search indexing" if do_index else None,
+                            *(["--maintenance-only", "--maintenance-batch-size", str(batch_size)] if index_only or migrate_only else []),
                         )
 
                 if not continuous:

archivebox/cli/archivebox_version.py

@@ -4,7 +4,10 @@ __package__ = "archivebox.cli"
 
 import sys
 import os
+import json
+import asyncio
 import platform
+import re
 from pathlib import Path
 from collections.abc import Iterable
 
@@ -29,8 +32,7 @@ def _format_binary_abspath(
 
     candidate_bases: tuple[tuple[Path, str], ...] = (
         (pwd, "./"),
-        (lib_dir, "LIB_DIR/"),
-        (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
+        (lib_dir, "ABXPKG_LIB_DIR/"),
         (personas_dir, "PERSONAS_DIR/"),
         (home, "~/"),
     )
@@ -47,8 +49,8 @@ def _format_binary_abspath(
             relative_str = relative.as_posix()
             if prefix == "./":
                 return "." if not relative_str else f"./{relative_str}"
-            if prefix == "LIB_DIR/":
-                return "LIB_DIR" if not relative_str else f"LIB_DIR/{relative_str}"
+            if prefix == "ABXPKG_LIB_DIR/":
+                return "ABXPKG_LIB_DIR" if not relative_str else f"ABXPKG_LIB_DIR/{relative_str}"
             if prefix == "PERSONAS_DIR/":
                 return "PERSONAS_DIR" if not relative_str else f"PERSONAS_DIR/{relative_str}"
             return "~" if not relative_str else f"~/{relative_str}"
@@ -59,10 +61,10 @@ def _format_binary_abspath(
 def _render_binary_abspath(abspath: str):
     from rich.text import Text
 
-    if abspath.startswith("LIB_DIR/"):
-        return Text.assemble(("LIB_DIR", "bright_blue"), (abspath.removeprefix("LIB_DIR"), "green"))
-    if abspath == "LIB_DIR":
-        return Text("LIB_DIR", style="bright_blue")
+    if abspath.startswith("ABXPKG_LIB_DIR/"):
+        return Text.assemble(("ABXPKG_LIB_DIR", "bright_blue"), (abspath.removeprefix("ABXPKG_LIB_DIR"), "green"))
+    if abspath == "ABXPKG_LIB_DIR":
+        return Text("ABXPKG_LIB_DIR", style="bright_blue")
     if abspath.startswith("PERSONAS_DIR/"):
         return Text.assemble(("PERSONAS_DIR", "medium_purple"), (abspath.removeprefix("PERSONAS_DIR"), "green"))
     if abspath == "PERSONAS_DIR":
@@ -78,6 +80,37 @@ def _render_binary_abspath(abspath: str):
     return Text(abspath, style="green")
 
 
+def _binary_record_matches_runtime(installed, lib_dir: Path) -> bool:
+    if not installed or not installed.is_valid or not installed.version:
+        return False
+    try:
+        abspath = Path(installed.abspath).expanduser().resolve(strict=False)
+        if not abspath.exists():
+            return False
+        if installed.binprovider not in {"", "env", "apt", "brew"}:
+            abspath.relative_to(lib_dir)
+    except (OSError, ValueError):
+        return False
+    return True
+
+
+def _binary_row_dedupe_key(
+    *,
+    display_name: str,
+    valid: bool,
+    version: str,
+    provider: str,
+    abspath: str,
+) -> tuple[str, str, str, str]:
+    if not valid:
+        return (display_name, "", "", "")
+    try:
+        resolved_abspath = Path(abspath).expanduser().resolve(strict=False).as_posix()
+    except Exception:
+        resolved_abspath = abspath
+    return (display_name, provider, version, resolved_abspath)
+
+
 @enforce_types
 def version(
     quiet: bool = False,
@@ -94,13 +127,16 @@ def version(
 
     from rich.panel import Panel
     from rich.console import Console
+    from rich.live import Live
+    from abx_dl.tables import binary_dependency_status, binary_dependency_table
 
     from archivebox.config import CONSTANTS
     from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
     from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID, IN_DOCKER
     from archivebox.config.paths import get_data_locations, get_code_locations
+    from archivebox.misc.checks import is_archivebox_source_root
     from archivebox.misc.logging_util import printable_folder_status
-    from archivebox.config.common import get_config
+    from archivebox.config.common import get_config, normalize_runtime_config
 
     console = Console()
     prnt = console.print
@@ -133,7 +169,7 @@ def version(
     try:
         DATA_DIR_STAT = CONSTANTS.DATA_DIR.stat()
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
             f"FS_UID={DATA_DIR_STAT.st_uid}:{DATA_DIR_STAT.st_gid}",
             f"FS_PERMS={config.OUTPUT_PERMISSIONS}",
             f"FS_ATOMIC={config.ENFORCE_ATOMIC_WRITES}",
@@ -141,7 +177,7 @@ def version(
         )
     except Exception:
         prnt(
-            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} PUID={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
+            f"EUID={os.geteuid()}:{os.getegid()} UID={RUNNING_AS_UID}:{RUNNING_AS_GID} ARCHIVEBOX_USER={ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}",
         )
 
     prnt(
@@ -154,7 +190,38 @@ def version(
     )
     prnt()
 
-    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    in_data_dir = (
+        not is_archivebox_source_root(CONSTANTS.DATA_DIR)
+        and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+        and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
+    )
+    if isinstance(binaries, str):
+        requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
+    else:
+        requested_names = {name for name in (binaries or ()) if name}
+
+    def binary_is_requested(logical_name: str, actual_name: str, display_name: str) -> bool:
+        return not requested_names or bool({logical_name, actual_name, display_name} & requested_names)
+
+    def plugin_may_have_requested_binary(plugin) -> bool:
+        if not requested_names:
+            return True
+        aliases = {plugin.name}
+        for required_binary in plugin.config.required_binaries:
+            raw_name = str(required_binary.name)
+            aliases.add(raw_name)
+            template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", raw_name)
+            if template_match:
+                config_key = template_match.group(1)
+                aliases.add(config_key)
+                aliases.add(config_key.removesuffix("_BINARY"))
+                default = plugin.config.properties.get(config_key, {}).get("default")
+                if default:
+                    aliases.add(str(default))
+        normalized_aliases = {alias.lower() for alias in aliases}
+        return bool({name.lower() for name in requested_names} & normalized_aliases)
+
+    if not in_data_dir:
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -174,93 +241,268 @@ def version(
                 subtitle="Full version info is only available when inside a collection [light_slate_blue]DATA DIR[/light_slate_blue]",
             ),
         )
+    prnt()
+
+    if not in_data_dir and not requested_names:
+        prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
+        prnt("", "[grey53]Dependency checks require an initialized collection DATA_DIR.[/grey53]")
+        prnt()
+        prnt("[deep_sky_blue3][i] Code locations:[/deep_sky_blue3]")
+        try:
+            for name, path in get_code_locations().items():
+                if isinstance(name, str) and isinstance(path, dict):
+                    prnt(printable_folder_status(name, path), overflow="ignore", crop=False)
+        except Exception as e:
+            prnt(f"  [red]Error getting code locations: {e}[/red]")
+        prnt()
+        prnt("[red][i] Data locations:[/red] (not in a data directory)")
         prnt()
         return []
 
     prnt("[pale_green1][i] Binary Dependencies:[/pale_green1]")
     failures = []
+    seen_failures: set[str] = set()
+    seen_rows: set[tuple[str, str, str, str]] = set()
 
-    # Setup Django before importing models
+    from archivebox.plugins.discovery import get_enabled_plugins
+    from abx_dl.config import get_required_binary_requests
+    from abx_dl.dependencies import resolve_binary_requests
+    from abx_dl.models import discover_plugins, filter_plugins
+    from abx_dl.orchestrator import create_bus
+    from abxpkg.binary_service import BinaryEvent, BinaryService
+
+    plugins = discover_plugins(runtime="archivebox")
+    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    enabled_plugin_names = set(enabled_plugins)
+    runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
+    derived_config: dict[str, object] = {}
+    db_binaries = {}
+    db_available = False
+    if in_data_dir:
+        try:
+            from archivebox.config.django import setup_django
+
+            setup_django()
+
+            from archivebox.machine.models import Machine, Binary
+
+            machine = Machine.current()
+            derived_config = normalize_runtime_config(machine.config, json_safe=False)
+            for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
+                if _binary_record_matches_runtime(binary, config.ABXPKG_LIB_DIR):
+                    db_binaries.setdefault(binary.name, binary)
+            db_available = True
+
+        except Exception as e:
+            prnt()
+            prnt("", f"[yellow]Warning: Could not query collection binary records; resolving through abxpkg: {e}[/yellow]")
+
+    declared_binary_specs: dict[str, dict[str, object]] = {}
+    for plugin_name, plugin in plugins.items():
+        if not plugin_may_have_requested_binary(plugin):
+            continue
+        plugin_requested = bool(requested_names)
+        plugin_enabled = plugin_name in enabled_plugin_names
+        logical_records = get_required_binary_requests(
+            plugin,
+            plugin.config.required_binaries,
+            overrides=runtime_config,
+            derived_overrides=derived_config,
+            run_output_dir=CONSTANTS.DATA_DIR,
+        )
+        actual_records = get_required_binary_requests(
+            plugin,
+            plugin.config.required_binaries,
+            overrides=runtime_config,
+            derived_overrides=derived_config,
+            run_output_dir=CONSTANTS.DATA_DIR,
+            logical_names=False,
+        )
+        for logical_record, actual_record in zip(logical_records, actual_records, strict=False):
+            logical_name = str(logical_record["name"])
+            actual_name = str(actual_record["name"])
+            display_name = Path(actual_name).expanduser().name if ("/" in actual_name or actual_name.startswith("~")) else logical_name
+            if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
+                continue
+            if not plugin_enabled and not requested_names:
+                continue
+            if _binary_record_matches_runtime(db_binaries.get(logical_name), config.ABXPKG_LIB_DIR):
+                continue
+            signature = json.dumps(actual_record, sort_keys=True, default=str)
+            declared_binary_specs.setdefault(signature, actual_record)
+
+    loaded_binaries: dict[str, BinaryEvent | None] = {}
+    if declared_binary_specs:
+        binary_bus = create_bus(name="ArchiveBoxVersionBinaryCheck")
+        BinaryService(binary_bus, auto_install=False, lib_dir=config.ABXPKG_LIB_DIR)
+
+        async def resolve_declared_binaries() -> dict[str, BinaryEvent | None]:
+            try:
+                return await resolve_binary_requests(binary_bus, declared_binary_specs)
+            finally:
+                await binary_bus.wait_until_idle()
+                await binary_bus.destroy(clear=False)
+
+        loaded_binaries = asyncio.run(resolve_declared_binaries())
+
+    rows: list[dict[str, object]] = []
+    any_rows = False
+    any_available = False
+    compact_paths = console.is_terminal
+    live_enabled = console.is_terminal
+    live_cm = Live(binary_dependency_table(rows), console=console, refresh_per_second=8) if live_enabled else None
+
+    def emit_row(row: dict[str, object]) -> None:
+        rows.append(row)
+        if live_cm is not None:
+            live_cm.update(binary_dependency_table(rows), refresh=True)
+
+    if live_cm is not None:
+        live_cm.start()
     try:
-        from archivebox.config.django import setup_django
-
-        setup_django()
-
-        from archivebox.machine.models import Machine, Binary
-
-        machine = Machine.current()
-
-        if isinstance(binaries, str):
-            requested_names = {name.strip() for name in binaries.split(",") if name.strip()}
-        else:
-            requested_names = {name for name in (binaries or ()) if name}
-
-        db_binaries: dict[str, Binary] = {}
-        for binary in Binary.objects.filter(machine=machine).order_by("name", "-modified_at"):
-            db_binaries.setdefault(binary.name, binary)
-
-        all_binary_names = sorted(requested_names or set(db_binaries.keys()))
-
-        if not all_binary_names:
-            prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
-        else:
-            any_available = False
-            compact_paths = console.is_terminal
-            for name in all_binary_names:
-                if requested_names and name not in requested_names:
+        for plugin_name, plugin in plugins.items():
+            if not plugin_may_have_requested_binary(plugin):
+                continue
+            plugin_requested = bool(requested_names)
+            plugin_enabled = plugin_name in enabled_plugin_names
+            logical_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+            )
+            actual_records = get_required_binary_requests(
+                plugin,
+                plugin.config.required_binaries,
+                overrides=runtime_config,
+                derived_overrides=derived_config,
+                run_output_dir=CONSTANTS.DATA_DIR,
+                logical_names=False,
+            )
+            for logical_record, actual_record in zip(logical_records, actual_records, strict=False):
+                logical_name = str(logical_record["name"])
+                actual_name = str(actual_record["name"])
+                display_name = Path(actual_name).expanduser().name if ("/" in actual_name or actual_name.startswith("~")) else logical_name
+                if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
                     continue
 
-                installed = db_binaries.get(name)
-                if installed and installed.is_valid:
-                    display_name = Path(name).expanduser().name if ("/" in name or name.startswith("~")) else name
+                installed = db_binaries.get(logical_name) if db_available else None
+                if _binary_record_matches_runtime(installed, config.ABXPKG_LIB_DIR):
+                    abspath = installed.abspath
+                    version_str = (installed.version or "unknown")[:15]
+                    provider = (installed.binprovider or "env")[:8]
+                    valid = True
+                elif not plugin_enabled and not requested_names:
+                    # `archivebox version` is expected to verify the active runtime, not
+                    # cold-load every optional plugin provider. Migration and status
+                    # checks often run with PLUGINS narrowed to a tiny set; resolving
+                    # disabled plugin binaries there can spend most of the command on
+                    # providers the current collection will never execute.
+                    continue
+                else:
+                    loaded = loaded_binaries[json.dumps(actual_record, sort_keys=True, default=str)]
+                    abspath = loaded.abspath if loaded is not None else ""
+                    version_str = str(loaded.version or "unknown")[:15] if loaded is not None else "unknown"
+                    provider = str(loaded.binprovider or "env")[:8] if loaded is not None else "env"
+                    valid = loaded is not None
+
+                any_rows = True
+                if valid:
                     display_path = (
                         _format_binary_abspath(
-                            installed.abspath,
+                            abspath,
                             pwd=Path.cwd(),
-                            lib_dir=config.LIB_DIR,
+                            lib_dir=config.ABXPKG_LIB_DIR,
                             personas_dir=CONSTANTS.PERSONAS_DIR,
                             home=Path.home(),
                         )
                         if compact_paths
-                        else installed.abspath
+                        else abspath
                     )
                     rendered_path = _render_binary_abspath(display_path) if compact_paths else display_path
-                    version_str = (installed.version or "unknown")[:15]
-                    provider = (installed.binprovider or "env")[:8]
-                    prnt(
-                        "",
-                        "[green]√[/green]",
-                        "",
-                        display_name.ljust(18),
-                        version_str.ljust(16),
-                        provider.ljust(8),
-                        rendered_path,
-                        overflow="ignore",
-                        crop=False,
-                    )
                     any_available = True
-                    continue
+                else:
+                    rendered_path = "[grey53]not installed[/grey53]"
+                    if plugin_enabled and display_name not in seen_failures:
+                        failures.append(display_name)
+                        seen_failures.add(display_name)
 
-                status = (
-                    "[grey53]not recorded[/grey53]" if name in requested_names and installed is None else "[grey53]not installed[/grey53]"
+                row_key = _binary_row_dedupe_key(
+                    display_name=display_name,
+                    valid=valid,
+                    version=version_str if valid else "-",
+                    provider=provider if valid else "-",
+                    abspath=abspath,
                 )
-                prnt("", "[red]X[/red]", "", name.ljust(18), status, overflow="ignore", crop=False)
-                failures.append(name)
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-            if not any_available:
-                prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
+                emit_row(
+                    {
+                        "plugin": plugin_name,
+                        "status": binary_dependency_status(enabled=plugin_enabled, valid=valid),
+                        "binary": display_name,
+                        "version": version_str if valid else "-",
+                        "provider": provider if valid else "-",
+                        "path": rendered_path,
+                        "style": "" if plugin_enabled else "dim",
+                    },
+                )
 
-        # Show hint if no binaries are installed yet
-        has_any_installed = Binary.objects.filter(machine=machine).exclude(abspath="").exists()
-        if not has_any_installed:
-            prnt()
-            prnt("", "[grey53]Run [green]archivebox install[/green] to detect and install dependencies.[/grey53]")
+        if db_available:
+            for binary_name, installed in db_binaries.items():
+                if requested_names and binary_name not in requested_names:
+                    continue
+                abspath = installed.abspath
+                version_str = (installed.version or "unknown")[:15]
+                provider = (installed.binprovider or "env")[:8]
+                row_key = _binary_row_dedupe_key(
+                    display_name=binary_name,
+                    valid=True,
+                    version=version_str,
+                    provider=provider,
+                    abspath=abspath,
+                )
+                if row_key in seen_rows:
+                    continue
+                seen_rows.add(row_key)
 
-    except Exception as e:
-        # Handle database errors gracefully (locked, missing, etc.)
-        prnt()
-        prnt("", f"[yellow]Warning: Could not query binaries from database: {e}[/yellow]")
-        prnt("", "[grey53]Run [green]archivebox init[/green] and [green]archivebox install[/green] to set up dependencies.[/grey53]")
+                display_path = (
+                    _format_binary_abspath(
+                        abspath,
+                        pwd=Path.cwd(),
+                        lib_dir=config.ABXPKG_LIB_DIR,
+                        personas_dir=CONSTANTS.PERSONAS_DIR,
+                        home=Path.home(),
+                    )
+                    if compact_paths
+                    else abspath
+                )
+                emit_row(
+                    {
+                        "plugin": "(database)",
+                        "status": binary_dependency_status(enabled=True, valid=True),
+                        "binary": binary_name,
+                        "version": version_str,
+                        "provider": provider,
+                        "path": _render_binary_abspath(display_path) if compact_paths else display_path,
+                        "style": "dim",
+                    },
+                )
+                any_rows = True
+                any_available = True
+    finally:
+        if live_cm is not None:
+            live_cm.stop()
+        else:
+            prnt(binary_dependency_table(rows))
+
+    if not any_rows:
+        prnt("", "[grey53]No required binaries declared for discovered plugins.[/grey53]")
+    elif not any_available:
+        prnt("", "[grey53]No binaries detected. Run [green]archivebox install[/green] to detect dependencies.[/grey53]")
 
     if not binaries:
         # Show code and data locations
@@ -319,9 +561,7 @@ def version(
 )
 @docstring(version.__doc__)
 def main(**kwargs):
-    failures = version(**kwargs)
-    if failures:
-        raise SystemExit(1)
+    version(**kwargs)
 
 
 if __name__ == "__main__":

archivebox/config/collection.py

@@ -194,6 +194,12 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
             decoded[key] = json.loads(raw)
         except (TypeError, ValueError):
             continue
+    if not str(decoded.get("BASE_URL") or "").strip():
+        from archivebox.config.common import base_url_from_legacy_server_config
+
+        legacy_base_url = base_url_from_legacy_server_config(decoded)
+        if legacy_base_url:
+            decoded["BASE_URL"] = legacy_base_url
     return decoded
 
 
@@ -320,7 +326,7 @@ def write_config_file(config: dict[str, str]) -> AttrDict:
         # validate the updated_config by attempting to re-parse it
         from archivebox.config.common import get_config
 
-        updated_config = get_config().as_dict()
+        updated_config = get_config(include_machine=False).as_dict()
     except BaseException:  # lgtm [py/catch-base-exception]
         # something went horribly wrong, revert to the previous version
         with open(f"{config_path}.bak", encoding="utf-8") as old:

archivebox/config/common.py

@@ -2,32 +2,32 @@ from __future__ import annotations
 
 __package__ = "archivebox.config"
 
+import inspect
 import json
 import os
 import re
 import secrets
-import sys
 import shutil
-import inspect
-from functools import lru_cache
+import sys
 from collections.abc import Mapping
 from datetime import timedelta
-from typing import Any, ClassVar, cast
+from functools import cache, lru_cache
 from pathlib import Path
-from urllib.parse import quote
+from typing import Any, ClassVar, cast
+from urllib.parse import quote, urlparse
 
-from rich.console import Console
+from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from django.db import DatabaseError
 from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
 from pydantic_settings import SettingsConfigDict
-from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from rich.console import Console
 
-from archivebox.config.configset import BaseConfigSet, IniConfigSettingsSource
-from archivebox.config.configset import COMPUTED_CONFIG_KEYS
+from archivebox.config.configset import COMPUTED_CONFIG_KEYS, BaseConfigSet, IniConfigSettingsSource
 
 from .constants import CONSTANTS
 from .ldap import LDAPConfig
-from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
 from .permissions import IN_DOCKER
+from .version import VERSION, get_BUILD_TIME, get_COMMIT_HASH
 
 ConfigOverrides = Mapping[str, object]
 ConfigPayload = dict[str, object]
@@ -67,6 +67,40 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
     return None
 
 
+def base_url_from_legacy_server_config(raw_config: Mapping[str, object]) -> str:
+    """Derive the current canonical URL from the pre-0.9 server settings."""
+
+    def normalize(value: object) -> str:
+        raw = str(value or "").strip()
+        if not raw:
+            return ""
+        parsed = urlparse(raw if "://" in raw else f"http://{raw}")
+        return f"{parsed.scheme}://{parsed.netloc}" if parsed.netloc else ""
+
+    for key in ("ARCHIVE_BASE_URL", "ADMIN_BASE_URL"):
+        candidate = normalize(raw_config.get(key))
+        if candidate:
+            return candidate
+
+    listen_host = str(raw_config.get("LISTEN_HOST") or "").strip()
+    if listen_host and listen_host.lower() != "archivebox.localhost:8000":
+        return normalize(listen_host)
+
+    csrf_origins = [normalize(origin) for origin in str(raw_config.get("CSRF_TRUSTED_ORIGINS") or "").split(",")]
+    csrf_origins = [origin for origin in csrf_origins if origin]
+    nondefault_csrf_origins = [origin for origin in csrf_origins if origin != "http://admin.archivebox.localhost:8000"]
+    if len(nondefault_csrf_origins) == 1:
+        return nondefault_csrf_origins[0]
+
+    allowed_hosts = [host.strip() for host in str(raw_config.get("ALLOWED_HOSTS") or "").split(",") if host.strip() != "*"]
+    if len(allowed_hosts) == 1:
+        return normalize(allowed_hosts[0])
+
+    if len(csrf_origins) == 1:
+        return csrf_origins[0]
+    return normalize(listen_host)
+
+
 def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str:
     for config in configs:
         if config is None:
@@ -176,6 +210,10 @@ def build_crawl_config_snapshot(
     plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
     effective = get_config(persona=persona, base_config=base_config)
     frozen = effective.for_crawl_frozen(persona=persona)
+    for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+        value = getattr(effective, key, None)
+        if value is not None:
+            frozen[key] = value
     if persona is not None:
         persona_config = persona.get_derived_config()
         for key in plugin_owned_keys - explicit_overrides:
@@ -185,6 +223,10 @@ def build_crawl_config_snapshot(
         resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
         resolved_payload = normalize_runtime_config(resolved)
         frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in ("BIND_ADDR", "BASE_URL", "CSRF_TRUSTED_ORIGINS", "SERVER_SECURITY_MODE"):
+            value = getattr(resolved, key, None)
+            if value is not None:
+                frozen[key] = value
         for key in plugin_owned_keys & explicit_overrides:
             if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
                 frozen[key] = resolved_payload[key]
@@ -238,18 +280,14 @@ class StorageConfig(BaseConfigSet):
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
-    # must be a short path due to unix path length restrictions for socket files (<100 chars)
+    # must be a short path due to unix path length restrictions for socket files (<90 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
     TMP_DIR: Path = Field(default=CONSTANTS.DEFAULT_TMP_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
+    # ABXPKG_LIB_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be able to contain executable binaries (up to 5GB size)
     # should not be a remote/network/FUSE mount for speed reasons, otherwise extractors will be slow
-    LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-
-    # LIB_BIN_DIR is an optional human-facing symlink convenience directory.
-    # Runtime lookup must use provider-specific paths under LIB_DIR instead.
-    LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    ABXPKG_LIB_DIR: Path = Field(default=CONSTANTS.DEFAULT_ABXPKG_LIB_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
     OUTPUT_PERMISSIONS: str = Field(default="644")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
@@ -268,6 +306,7 @@ class ServerConfig(BaseConfigSet):
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
     SERVER_SECURITY_MODES: ClassVar[tuple[str, ...]] = (
+        "auto",
         "safe-subdomains-fullreplay",
         "safe-onedomain-nojsreplay",
         "unsafe-onedomain-noadmin",
@@ -279,7 +318,7 @@ class ServerConfig(BaseConfigSet):
     BASE_URL: str = Field(default="")
     ALLOWED_HOSTS: str = Field(default="*")
     CSRF_TRUSTED_ORIGINS: str = Field(default="")
-    SERVER_SECURITY_MODE: str = Field(default="safe-subdomains-fullreplay")
+    SERVER_SECURITY_MODE: str = Field(default="auto")
 
     SNAPSHOTS_PER_PAGE: int = Field(default=50, ge=1)
     FOOTER_INFO: str = Field(
@@ -304,11 +343,21 @@ class ServerConfig(BaseConfigSet):
 
     @property
     def USES_SUBDOMAIN_ROUTING(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay"
+        if self.SERVER_SECURITY_MODE == "safe-subdomains-fullreplay":
+            return True
+        if self.SERVER_SECURITY_MODE != "auto":
+            return False
+
+        base_host = (urlparse(self.BASE_URL if "://" in self.BASE_URL else f"//{self.BASE_URL}").hostname or "").lower()
+        if base_host:
+            return base_host.endswith(".localhost")
+
+        bind_host = (urlparse(f"//{self.BIND_ADDR}").hostname or "").lower()
+        return bind_host in {"", "0.0.0.0", "127.0.0.1", "::", "::1", "localhost"}
 
     @property
     def ENABLES_FULL_JS_REPLAY(self) -> bool:
-        return self.SERVER_SECURITY_MODE in (
+        return self.USES_SUBDOMAIN_ROUTING or self.SERVER_SECURITY_MODE in (
             "safe-subdomains-fullreplay",
             "unsafe-onedomain-noadmin",
             "danger-onedomain-fullreplay",
@@ -324,7 +373,7 @@ class ServerConfig(BaseConfigSet):
 
     @property
     def SHOULD_NEUTER_RISKY_REPLAY(self) -> bool:
-        return self.SERVER_SECURITY_MODE == "safe-onedomain-nojsreplay"
+        return self.SERVER_SECURITY_MODE in ("auto", "safe-onedomain-nojsreplay")
 
     @property
     def IS_UNSAFE_MODE(self) -> bool:
@@ -346,7 +395,16 @@ class DatabaseConfig(BaseConfigSet):
     toml_section_header: str = "DATABASE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
+    DATABASE_ENGINE: str = Field(
+        default="sqlite",
+        alias="ARCHIVEBOX_DATABASE_ENGINE",
+        pattern=r"(?i)^(sqlite|postgres|postgresql)$",
+    )
     DATABASE_NAME: str = Field(default=str(CONSTANTS.DATABASE_FILE), alias="ARCHIVEBOX_DATABASE_NAME")
+    DATABASE_HOST: str = Field(default="127.0.0.1", alias="ARCHIVEBOX_DATABASE_HOST")
+    DATABASE_PORT: int = Field(default=5432, alias="ARCHIVEBOX_DATABASE_PORT", ge=1, le=65535)
+    DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
+    DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
     SQLITE_JOURNAL_MODE: str = Field(
         default="WAL",
         alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
@@ -488,7 +546,7 @@ class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
+    SEARCH_BACKEND_ENGINE: str = Field(default="sonic", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
 
 def _plugin_user_config_value(value: Any) -> str:
@@ -503,6 +561,32 @@ def _plugin_user_config(config: Mapping[str, object]) -> dict[str, str]:
     return {key: _plugin_user_config_value(value) for key, value in config.items()}
 
 
+def _plugin_input_config(config: Mapping[str, object]) -> dict[str, object]:
+    """Return raw config inputs understood by core or plugin schemas."""
+    input_names = _archivebox_config_input_names()
+    return {str(key): value for key, value in config.items() if str(key) in input_names}
+
+
+def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
+    """Return canonical ``*_ENABLED`` keys explicitly provided by the user.
+
+    ``PLUGINS`` is a selector, not an override for concrete plugin enable flags.
+    If a user passes ``PLUGINS=wget`` and ``SAVE_WGET=False`` together, the
+    alias-resolved ``WGET_ENABLED=False`` must survive the model validator that
+    expands ``PLUGINS`` into concrete booleans.
+    """
+    provided = {str(key) for key in config}
+    explicit: set[str] = set()
+    for key, prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).items():
+        if not key.endswith("_ENABLED") or not isinstance(prop, Mapping):
+            continue
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        input_names = {key, *(str(alias) for alias in aliases if str(alias).strip())} if isinstance(aliases, list) else {key}
+        if provided & input_names:
+            explicit.add(key)
+    return explicit
+
+
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
     from archivebox.plugins.discovery import discover_plugin_configs
 
@@ -532,6 +616,10 @@ def _archivebox_config_input_names() -> set[str]:
     for field in ArchiveBoxConfig.model_fields.values():
         if isinstance(field.alias, str):
             names.add(field.alias)
+    for prop in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS).values():
+        aliases = prop.get("x-aliases") if isinstance(prop, Mapping) else None
+        if isinstance(aliases, list):
+            names.update(str(alias) for alias in aliases if str(alias).strip())
     return names
 
 
@@ -600,7 +688,7 @@ class ArchiveBoxBaseConfig(
     @classmethod
     def _plugin_field_scope(cls, key: str) -> str | None:
         scope = None
-        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        for schema in PLUGIN_CONFIG_SCHEMAS.values():
             properties = schema.get("properties") if isinstance(schema, dict) else None
             if not isinstance(properties, dict) or key not in properties:
                 continue
@@ -612,7 +700,7 @@ class ArchiveBoxBaseConfig(
         return scope
 
     @classmethod
-    @lru_cache(maxsize=None)
+    @cache
     def scope_for_key(cls, key: str) -> str:
         for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
             properties = schema.get("properties") if isinstance(schema, dict) else None
@@ -643,11 +731,10 @@ class ArchiveBoxBaseConfig(
         runtime_derived_keys = {
             "ABX_INSTALL_CACHE",
             "ACTIVE_PERSONA",
-            "CHROME_DOWNLOADS_DIR",
-            "CHROME_USER_DATA_DIR",
             "CRAWL_DIR",
             "DEFAULT_PERSONA",
             "EXTRA_CONTEXT",
+            "PERSONAS_DIR",
             "SNAP_DIR",
         }
         return frozenset(
@@ -657,7 +744,7 @@ class ArchiveBoxBaseConfig(
     def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
         keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
         payload = self.model_dump(mode="json")
-        return {key: payload[key] for key in keys if payload.get(key) is not None}
+        return {key: payload[key] for key in keys if payload.get(key, None) is not None}
 
     def for_crawl(self) -> dict[str, Any]:
         """Config scoped to crawl execution, without runtime object overlays."""
@@ -693,9 +780,9 @@ class ArchiveBoxBaseConfig(
         config["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
         scope_by_key = type(self)._scope_by_key()
         model_fields = type(self).model_fields
-        # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model
-        # validation to derive the selected backend's *_ENABLED flag. Hooks
-        # only receive the backend-local flags, never the selector itself.
+        # ArchiveBox-only selectors are consumed during model validation to
+        # derive hook-owned *_ENABLED flags. They are stripped below after
+        # crawl/snapshot overlays have had a chance to apply.
         config.pop("SEARCH_BACKEND_ENGINE", None)
         if persona is not None:
             for key, value in persona.get_derived_config().items():
@@ -717,6 +804,11 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        # Hooks should only see concrete plugin-local flags, never the
+        # ArchiveBox selectors used to derive them.
+        config.pop("PLUGINS", None)
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+
         if extra_context:
             context: dict[str, Any] = {}
             if config.get("EXTRA_CONTEXT"):
@@ -731,31 +823,46 @@ class ArchiveBoxBaseConfig(
 
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
-        lib_dir = self.LIB_DIR.expanduser()
+        lib_dir = self.ABXPKG_LIB_DIR.expanduser()
         if not lib_dir.is_absolute():
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
-        self.LIB_DIR = lib_dir.resolve()
-
-        lib_bin_dir = self.LIB_BIN_DIR.expanduser()
-        if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
-            lib_bin_dir = self.LIB_DIR / "bin"
-        elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
-        self.LIB_BIN_DIR = lib_bin_dir.resolve()
+        self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
         return self
 
     @model_validator(mode="after")
     def derive_plugin_enabled_config(self):
+        self._derive_plugin_enabled_config(respect_current_enabled=False)
+        return self
+
+    def _derive_plugin_enabled_config(self, *, respect_current_enabled: bool) -> None:
         plugin_names = _normalize_plugins_config_value(self.PLUGINS)
-        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        enabled_config_keys = _plugin_enabled_config_keys()
+        disabled_plugins = []
+        if respect_current_enabled:
+            disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
+        selected_plugin_roots = plugin_names
+        if selected_plugin_roots:
+            from abx_dl.models import discover_plugins, filter_plugins
+
+            selected_plugins = set(
+                filter_plugins(
+                    discover_plugins(runtime="archivebox"),
+                    sorted(selected_plugin_roots),
+                    include_providers=True,
+                    disabled_names=disabled_plugins,
+                ),
+            )
+        else:
+            selected_plugins = set()
         search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
         if search_backend:
             selected_plugins.add(f"search_backend_{search_backend}")
-        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+        if respect_current_enabled:
+            selected_plugins.difference_update(disabled_plugins)
+        for plugin_name, enabled_key in enabled_config_keys.items():
             if plugin_names or plugin_name in selected_plugins:
                 setattr(self, enabled_key, plugin_name in selected_plugins)
-        return self
 
 
 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
@@ -812,23 +919,6 @@ def _plugin_enabled_config_keys() -> dict[str, str]:
     return enabled_keys
 
 
-def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
-    selected = set(plugin_names)
-    pending = list(selected)
-    while pending:
-        plugin_name = pending.pop()
-        schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
-        required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
-        if not isinstance(required_plugins, list):
-            continue
-        for required_plugin in required_plugins:
-            required_plugin_name = str(required_plugin).strip().lower()
-            if required_plugin_name and required_plugin_name not in selected:
-                selected.add(required_plugin_name)
-                pending.append(required_plugin_name)
-    return selected
-
-
 def get_live_config_url(key: str) -> str:
     return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
 
@@ -952,8 +1042,12 @@ def get_request_config(request: Any, *, resolve_plugins: bool = False) -> Archiv
     request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
     if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
         request_config = get_config(resolve_plugins=resolve_plugins)
-        request.archivebox_config = request_config
         request._archivebox_config_resolves_plugins = resolve_plugins
+    if request_config.SERVER_SECURITY_MODE == "auto":
+        request_host = (urlparse(f"//{request.get_host()}").hostname or "").lower().rstrip(".")
+        effective_mode = "safe-subdomains-fullreplay" if request_host.endswith(".localhost") else "safe-onedomain-nojsreplay"
+        request_config = request_config.model_copy(update={"SERVER_SECURITY_MODE": effective_mode})
+    request.archivebox_config = request_config
     return request_config
 
 
@@ -993,17 +1087,20 @@ def get_config(
                 from archivebox.machine.models import Machine
 
                 machine = Machine.current()
-        except Exception:
+        except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
             machine = None
 
     if persona is None and crawl is not None:
         persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
+    config_data["PERSONAS_DIR"] = str(CONSTANTS.PERSONAS_DIR)
     base_config_payload: ConfigPayload = {}
+    base_config_model = ArchiveBoxConfig()
+
     if crawl_config_base:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
         config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
     elif base_config is not None:
@@ -1014,11 +1111,16 @@ def get_config(
         config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
         config_data.update(
-            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+            normalize_runtime_config(base_config_model.model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
         )
-        legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
+        legacy_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ}
+        legacy_permissions = permissions_from_legacy_public_flags(legacy_config)
         if legacy_permissions:
             config_data["PERMISSIONS"] = legacy_permissions
+        if not str(config_data.get("BASE_URL") or "").strip():
+            legacy_base_url = base_url_from_legacy_server_config(legacy_config)
+            if legacy_base_url:
+                config_data["BASE_URL"] = legacy_base_url
 
     scope_overrides: ConfigPayload = {}
 
@@ -1027,7 +1129,7 @@ def get_config(
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")),
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,
@@ -1066,22 +1168,29 @@ def get_config(
     archivebox_scope_overrides = {key: value for key, value in scope_overrides.items() if key in _archivebox_config_input_names()}
     config_data.update(archivebox_scope_overrides)
 
+    explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
         plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
+        crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
+        # A frozen crawl-level PLUGINS selector is the exact extractor set for
+        # this crawl. Env/file *_ENABLED values are ambient defaults and must
+        # not be re-applied after validation, or a later process env can change
+        # which hooks an already-queued crawl runs.
+        if not crawl_selected_plugins:
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(os.environ))
+        explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(scope_overrides))
         plugin_user_config = _plugin_user_config(
             {
-                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
+                **_plugin_input_config(os.environ),
                 **scope_overrides,
             },
         )
         if not crawl_config_base:
+            file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+            explicit_plugin_enabled_keys.update(_explicit_plugin_enabled_keys(file_config))
             plugin_user_config = {
-                **normalize_runtime_config(
-                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
-                    exclude_runtime_derived=True,
-                    json_safe=False,
-                ),
+                **_plugin_user_config(_plugin_input_config(file_config)),
                 **plugin_user_config,
             }
         plugin_sections = resolve_plugin_configs(
@@ -1106,6 +1215,13 @@ def get_config(
         if crawl_config_base:
             config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
+        if crawl_selected_plugins:
+            # Crawl PLUGINS is a frozen selector. Drop ambient enable flags from
+            # env/file/defaults before model validation so the selected plugin
+            # set is derived from the crawl, not from the worker process that
+            # happens to resume it.
+            for enabled_key in _plugin_enabled_config_keys().values():
+                config_data.pop(enabled_key, None)
 
     # Decode JSON-encoded complex values (dict/list fields) that came from
     # string-only sources before validation. ``IniConfigSettingsSource`` does
@@ -1131,14 +1247,21 @@ def get_config(
             )
 
     config = ArchiveBoxConfig.model_validate(config_data)
+    if config.SERVER_SECURITY_MODE == "auto":
+        base_host = (urlparse(config.BASE_URL if "://" in config.BASE_URL else f"//{config.BASE_URL}").hostname or "").lower()
+        if base_host.endswith(".localhost"):
+            config = config.model_copy(update={"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay"})
+    for key in explicit_plugin_enabled_keys:
+        if key in config_data:
+            setattr(config, key, config_data[key])
+    if config.PLUGINS:
+        config._derive_plugin_enabled_config(respect_current_enabled=True)
     if redact_sensitive:
         for key in type(config).model_fields:
             value = config[key]
             if is_sensitive_config_key(key) and value not in (None, ""):
                 setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
-    os.environ["LIB_DIR"] = str(config.LIB_DIR)
-    os.environ["LIB_BIN_DIR"] = str(config.LIB_BIN_DIR)
-    os.environ["ABXPKG_LIB_DIR"] = str(config.LIB_DIR)
+    os.environ["ABXPKG_LIB_DIR"] = str(config.ABXPKG_LIB_DIR)
     archiving_warning_key = (config.TIMEOUT, config.USE_COLOR)
     if archiving_warning_key not in _WARNED_ARCHIVING_CONFIGS:
         config.warn_if_invalid()

archivebox/config/configset.py

@@ -43,7 +43,7 @@ def _read_ini_config_cached(config_path_str: str) -> dict[str, Any]:
     config_path = Path(config_path_str)
     try:
         mtime = config_path.stat().st_mtime
-    except FileNotFoundError:
+    except (FileNotFoundError, PermissionError):
         return {}
     cache_key = (str(config_path), mtime)
     cached = _INI_CACHE.get(cache_key)

archivebox/config/constants.py

@@ -36,8 +36,8 @@ from .permissions import (
     IN_DOCKER,
     RUNNING_AS_UID,
     RUNNING_AS_GID,
-    DEFAULT_PUID,
-    DEFAULT_PGID,
+    DEFAULT_UID,
+    DEFAULT_GID,
     ARCHIVEBOX_USER,
     ARCHIVEBOX_GROUP,
 )
@@ -66,8 +66,8 @@ class ConstantsDict:
     ARCHIVEBOX_GROUP: int = ARCHIVEBOX_GROUP
     RUNNING_AS_UID: int = RUNNING_AS_UID
     RUNNING_AS_GID: int = RUNNING_AS_GID
-    DEFAULT_PUID: int = DEFAULT_PUID
-    DEFAULT_PGID: int = DEFAULT_PGID
+    DEFAULT_UID: int = DEFAULT_UID
+    DEFAULT_GID: int = DEFAULT_GID
     IS_INSIDE_VENV: bool = sys.prefix != sys.base_prefix
 
     # Source code dirs
@@ -113,12 +113,10 @@ class ConstantsDict:
     TMP_DIR_NAME: str = "tmp"
     DEFAULT_TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME / MACHINE_ID  # ./data/tmp/abc3244323
 
-    LIB_DIR_NAME: str = "lib"
-    DEFAULT_LIB_DIR: Path = _env_path(
-        "LIB_DIR",
-        user_config_path("abx") / LIB_DIR_NAME,
+    DEFAULT_ABXPKG_LIB_DIR: Path = _env_path(
+        "ABXPKG_LIB_DIR",
+        user_config_path("abx") / "lib",
     )
-    DEFAULT_LIB_BIN_DIR: Path = _env_path("LIB_BIN_DIR", DEFAULT_LIB_DIR / "bin")
 
     RESERVED_ARCHIVE_DIR_NAMES: frozenset[str] = frozenset(
         (
@@ -241,7 +239,7 @@ class ConstantsDict:
             SOURCES_DIR_NAME,
             LOGS_DIR_NAME,
             CACHE_DIR_NAME,
-            LIB_DIR_NAME,
+            "lib",
             TMP_DIR_NAME,
             PERSONAS_DIR_NAME,
             CUSTOM_TEMPLATES_DIR_NAME,
@@ -295,5 +293,3 @@ CONSTANTS = ConstantsDict
 CONSTANTS_CONFIG = AttrDict({key: value for key, value in CONSTANTS.__dict__.items() if key.isupper() and not key.startswith("_")})
 
 # add all key: values to globals() for easier importing, e.g.:
-# from archivebox.config.constants import IS_ROOT, PERSONAS_DIR, ...
-# globals().update(CONSTANTS)

archivebox/config/django.py

@@ -2,19 +2,17 @@ __package__ = "archivebox.config"
 
 import os
 import sys
-import subprocess
-
-from datetime import datetime, timezone
-
-from rich.console import Console
+from datetime import UTC, datetime
 
 import django
 import django.db
+from django.core.exceptions import ImproperlyConfigured
+from rich.console import Console
 
 from archivebox.misc import logging
 
-from .constants import CONSTANTS
 from .common import get_config
+from .constants import CONSTANTS
 
 CONFIG = get_config()
 
@@ -31,32 +29,40 @@ logging.CONSOLE = CONSOLE
 DJANGO_SET_UP = False
 
 
-def setup_django(check_db=False, in_memory_db=False) -> None:
+def setup_django(check_db=False) -> None:
     from rich.panel import Panel
 
+    from archivebox.misc.checks import check_not_inside_source_dir
+
     global DJANGO_SET_UP
 
     if DJANGO_SET_UP:
-        # raise Exception('django is already set up!')
         # TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
         return
 
+    check_not_inside_source_dir(CONSTANTS.DATA_DIR)
+
+    # SQLite creates index.sqlite3 during django.setup()/migrate. Apply the
+    # ArchiveBox file-mode policy before any DB connection can create the file,
+    # otherwise a permissive parent umask can expose a just-created DB until a
+    # later chmod runs.
+    os.umask(0o777 - (int(CONFIG.OUTPUT_PERMISSIONS, base=8) | 0o111))
+
     # Third-party patches are only needed once Django/apps are about to load.
     # Keeping them out of archivebox.__init__ avoids paying Django/Daphne setup
     # cost for cheap CLI startup paths like `archivebox <cmd> --help`.
     import archivebox.misc.monkey_patches  # noqa: F401
-
-    from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, SudoPermission
+    from archivebox.config.permissions import ARCHIVEBOX_GROUP, ARCHIVEBOX_USER, IS_ROOT, SudoPermission
 
     # if running as root, chown the data dir to the archivebox user to make sure it's accessible to the archivebox user
     if IS_ROOT and ARCHIVEBOX_USER != 0:
         with SudoPermission(uid=0):
             # running as root is a special case where it's ok to be a bit slower
             # make sure data dir is always owned by the correct user
-            subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(CONSTANTS.DATA_DIR)], stderr=subprocess.DEVNULL)
+            os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
             if CONSTANTS.DATA_DIR.exists():
                 for child in CONSTANTS.DATA_DIR.iterdir():
-                    subprocess.run(["chown", f"{ARCHIVEBOX_USER}:{ARCHIVEBOX_GROUP}", str(child)], stderr=subprocess.DEVNULL)
+                    os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
 
     # Suppress the "database access during app initialization" warning
     # This warning can be triggered during django.setup() but is safe to ignore
@@ -72,40 +78,30 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
     try:
         from django.core.management import call_command
 
-        if in_memory_db:
-            raise Exception("dont use this anymore")
-
-            # some commands dont store a long-lived sqlite3 db file on disk.
-            # in those cases we create a temporary in-memory db and run the migrations
-            # immediately to get a usable in-memory-database at startup
-            os.environ.setdefault("ARCHIVEBOX_DATABASE_NAME", ":memory:")
+        # Initialize the configured file-based database without running
+        # migrations automatically; `archivebox init` owns migrations.
+        try:
             django.setup()
+        # User config and import errors are reported through one CLI message.
+        except (ImproperlyConfigured, django.db.Error, ImportError, OSError, RuntimeError, ValueError) as e:
+            is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version"))
+            if not is_using_meta_cmd:
+                # show error message to user only if they're not running a meta command / just trying to get help
+                STDERR.print()
+                STDERR.print(
+                    Panel(
+                        f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n",
+                        title="\n\n[red][X] Error while trying to load database![/red]",
+                        subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]",
+                        expand=False,
+                        style="bold red",
+                    ),
+                )
+                STDERR.print()
+                import traceback
 
-            call_command("migrate", interactive=False, verbosity=0)
-        else:
-            # Otherwise use default sqlite3 file-based database and initialize django
-            # without running migrations automatically (user runs them manually by calling init)
-            try:
-                django.setup()
-            except Exception as e:
-                is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ("help", "version", "--help", "--version"))
-                if not is_using_meta_cmd:
-                    # show error message to user only if they're not running a meta command / just trying to get help
-                    STDERR.print()
-                    STDERR.print(
-                        Panel(
-                            f"\n[red]{e.__class__.__name__}[/red]: [yellow]{e}[/yellow]\nPlease check your config and [blue]DATA_DIR[/blue] permissions.\n",
-                            title="\n\n[red][X] Error while trying to load database![/red]",
-                            subtitle="[grey53]NO WRITES CAN BE PERFORMED[/grey53]",
-                            expand=False,
-                            style="bold red",
-                        ),
-                    )
-                    STDERR.print()
-                    import traceback
-
-                    traceback.print_exc()
-                return
+                traceback.print_exc()
+            return
 
         from archivebox.core.settings_logging import ERROR_LOG as DEFAULT_ERROR_LOG
 
@@ -113,13 +109,13 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
         error_log = DEFAULT_ERROR_LOG
         with open(error_log, "a", encoding="utf-8") as f:
             command = " ".join(sys.argv)
-            ts = datetime.now(timezone.utc).strftime("%Y-%m-%d__%H:%M:%S")
+            ts = datetime.now(UTC).strftime("%Y-%m-%d__%H:%M:%S")
             config = get_config()
             f.write(f"\n> {command}; TS={ts} VERSION={CONSTANTS.VERSION} IN_DOCKER={config.IN_DOCKER} IS_TTY={config.IS_TTY}\n")
 
         if check_db:
             # make sure the data dir is owned by a non-root user
-            if CONSTANTS.DATA_DIR.stat().st_uid == 0:
+            if CONSTANTS.DATA_DIR.stat().st_uid == 0 and not (IS_ROOT and ARCHIVEBOX_USER == 0):
                 STDERR.print("[red][X] Error: ArchiveBox DATA_DIR cannot be owned by root![/red]")
                 STDERR.print(f"    {CONSTANTS.DATA_DIR}")
                 STDERR.print()
@@ -145,23 +141,14 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
             for conn in connections.all():
                 conn.close_if_unusable_or_obsolete()
 
-            sql_index_path = CONSTANTS.DATABASE_FILE
-            assert os.access(sql_index_path, os.F_OK), (
-                f"No database file {sql_index_path} found in: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)"
+            from archivebox.misc.db import database_display_location, database_exists
+
+            assert database_exists(), (
+                f"No database {database_display_location()} found for: {CONSTANTS.DATA_DIR} (Are you in an ArchiveBox collection directory?)"
             )
 
-            # https://docs.pydantic.dev/logfire/integrations/django/ Logfire Debugging
-            # if settings.DEBUG_LOGFIRE:
-            #     from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
-            #     SQLite3Instrumentor().instrument()
-
-            #     import logfire
-
-            #     logfire.configure()
-            #     logfire.instrument_django(is_sql_commentor_enabled=True)
-            #     logfire.info(f'Started ArchiveBox v{CONSTANTS.VERSION}', argv=sys.argv)
-
     except KeyboardInterrupt:
+        DJANGO_SET_UP = False
         raise
 
     DJANGO_SET_UP = True

archivebox/config/paths.py

@@ -5,7 +5,6 @@ import socket
 import hashlib
 import tempfile
 import platform
-import subprocess
 from pathlib import Path
 from functools import cache
 from datetime import datetime
@@ -20,6 +19,8 @@ if TYPE_CHECKING:
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
 DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
+MAX_TMP_SOCKET_URL_LENGTH = 90
+SUPERVISORD_SOCKET_FILENAME = "supervisord.sock"
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -57,9 +58,11 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
     collection_id = hashlib.sha256(f"{machine_id}:{collection_path}@{creation_date}".encode()).hexdigest()[:8]
 
     try:
-        # only persist collection_id file if we already have an index.sqlite3 file present
+        # only persist collection_id file if this dir already looks like a real collection
+        # (has an index.sqlite3, or an ArchiveBox.conf when the DB lives in postgres),
         # otherwise we might be running in a directory that is not a collection, no point creating cruft files
-        collection_is_active = os.path.isfile(DATABASE_FILE) and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
+        collection_marker = os.path.isfile(DATABASE_FILE) or os.path.isfile(DATA_DIR / "ArchiveBox.conf")
+        collection_is_active = collection_marker and os.path.isdir(ARCHIVE_DIR) and os.access(DATA_DIR, os.W_OK)
         if collection_is_active or force_create:
             collection_id_file.write_text(collection_id)
 
@@ -67,9 +70,9 @@ def _get_collection_id(DATA_DIR=DATA_DIR, force_create=False) -> str:
             if IS_ROOT:
                 with SudoPermission(uid=0):
                     if ARCHIVEBOX_USER == 0:
-                        subprocess.run(["chmod", "777", str(collection_id_file)])
+                        collection_id_file.chmod(0o777)
                     else:
-                        subprocess.run(["chown", str(ARCHIVEBOX_USER), str(collection_id_file)])
+                        os.chown(collection_id_file, ARCHIVEBOX_USER, -1)
     except (OSError, FileNotFoundError, PermissionError):
         pass
     return collection_id
@@ -120,14 +123,14 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
     try:
         with SudoPermission(uid=uid, fallback=fallback):
             test_file.exists()
-            test_file.write_text(f"Checking if PUID={uid} PGID={gid} can write to dir")
+            test_file.write_text(f"Checking if uid={uid} gid={gid} can write to dir")
             test_file.unlink()
             return True
     except (OSError, PermissionError):
         if chown:
             # try fixing it using sudo permissions
             with SudoPermission(uid=uid, fallback=fallback):
-                subprocess.run(["chown", f"{uid}:{gid}", str(dir_path)], stderr=subprocess.DEVNULL)
+                os.chown(dir_path, uid, gid)
             return dir_is_writable(dir_path, uid=uid, gid=gid, fallback=fallback, chown=False)
     return False
 
@@ -176,33 +179,45 @@ def create_and_chown_dir(dir_path: Path) -> None:
 
 
 def tmp_dir_socket_path_is_short_enough(dir_path: Path) -> bool:
-    socket_file = dir_path.absolute().resolve() / "supervisord.sock"
-    return len(f"file://{socket_file}") <= 96
+    socket_file = dir_path.absolute().resolve() / SUPERVISORD_SOCKET_FILENAME
+    return len(f"file://{socket_file}") < MAX_TMP_SOCKET_URL_LENGTH
+
+
+def tmp_dir_candidates(config: "ArchiveBoxConfig") -> list[Path]:
+    from archivebox.config.constants import CONSTANTS
+
+    collection_id = get_collection_id()
+    collection_id_short = collection_id[:4]
+    system_tmp_dir = Path(tempfile.gettempdir())
+    candidates = [
+        config.TMP_DIR,  # <user-specified>
+        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
+        Path("/var/run/archivebox") / collection_id,
+        Path("/tmp") / "archivebox" / collection_id,
+        Path("~/.tmp/archivebox").expanduser() / collection_id,
+        system_tmp_dir / "archivebox" / collection_id,
+        system_tmp_dir / "archivebox" / collection_id_short,
+        system_tmp_dir / "abx" / collection_id_short,
+    ]
+    seen = set()
+    unique_candidates = []
+    for path in candidates:
+        path_key = str(path.expanduser().absolute())
+        if path_key in seen:
+            continue
+        seen.add(path_key)
+        unique_candidates.append(path)
+    return unique_candidates
 
 
 def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
-    from archivebox.config.constants import CONSTANTS
     from archivebox.config.common import get_config
     from archivebox.misc.checks import check_tmp_dir
 
     config = config or get_config(**config_kwargs)
-    # try a few potential directories in order of preference
-    CANDIDATES = [
-        config.TMP_DIR,  # <user-specified>
-        CONSTANTS.DEFAULT_TMP_DIR,  # ./data/tmp/<machine_id>
-        Path("/var/run/archivebox") / get_collection_id(),  # /var/run/archivebox/abc5d8512
-        Path("/tmp") / "archivebox" / get_collection_id(),  # /tmp/archivebox/abc5d8512
-        Path("~/.tmp/archivebox").expanduser() / get_collection_id(),  # ~/.tmp/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id(),  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d8512
-        Path(tempfile.gettempdir())
-        / "archivebox"
-        / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/archivebox/abc5d
-        Path(tempfile.gettempdir()) / "abx" / get_collection_id()[:4],  # /var/folders/qy/6tpfrpx100j1t4l312nz683m0000gn/T/abx/abc5
-    ]
+    candidates = tmp_dir_candidates(config)
     fallback_candidate = None
-    for candidate in CANDIDATES:
+    for candidate in candidates:
         try:
             create_and_chown_dir(candidate)
         except Exception:
@@ -231,7 +246,7 @@ def get_or_create_working_tmp_dir(autofix=True, quiet=True, config: "ArchiveBoxC
         return fallback_candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable TMP_DIR, tried {candidates}!")
 
 
 def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -240,8 +255,8 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
 
     config = config or get_config(**config_kwargs)
 
-    # LIB_DIR is either the shared platformdirs default or an explicit env/config override.
-    CANDIDATES = [config.LIB_DIR]
+    # ABXPKG_LIB_DIR is either the shared platformdirs default or an explicit env/config override.
+    CANDIDATES = [config.ABXPKG_LIB_DIR]
 
     for candidate in CANDIDATES:
         try:
@@ -249,12 +264,30 @@ def get_or_create_working_lib_dir(autofix=True, quiet=False, config: "ArchiveBox
         except Exception:
             pass
         if check_lib_dir(candidate, throw=False, quiet=True, must_exist=True):
-            if autofix and config.LIB_DIR != candidate:
-                os.environ["LIB_DIR"] = str(candidate)
+            if autofix and config.ABXPKG_LIB_DIR != candidate:
+                os.environ["ABXPKG_LIB_DIR"] = str(candidate)
             return candidate
 
     if not quiet:
-        raise OSError(f"ArchiveBox is unable to find a writable LIB_DIR, tried {CANDIDATES}!")
+        raise OSError(f"ArchiveBox is unable to find a writable ABXPKG_LIB_DIR, tried {CANDIDATES}!")
+
+
+def _sql_index_location() -> dict:
+    from archivebox.misc.db import database_display_location, database_exists, is_postgres
+
+    if is_postgres():
+        return {
+            "path": database_display_location(),
+            "enabled": True,
+            "is_valid": database_exists(),
+            "is_mount": False,
+        }
+    return {
+        "path": DATABASE_FILE.resolve(),
+        "enabled": True,
+        "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
+        "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
+    }
 
 
 def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs):
@@ -283,12 +316,7 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)
                 and os.access(CONSTANTS.CONFIG_FILE, os.W_OK),
             },
-            "SQL_INDEX": {
-                "path": DATABASE_FILE.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isfile(DATABASE_FILE) and os.access(DATABASE_FILE, os.R_OK) and os.access(DATABASE_FILE, os.W_OK),
-                "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
-            },
+            "SQL_INDEX": _sql_index_location(),
             "ARCHIVE_DIR": {
                 "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                 "enabled": True,
@@ -329,13 +357,11 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
             "TMP_DIR": {
                 "path": tmp_dir.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(tmp_dir) and os.access(tmp_dir, os.R_OK) and os.access(tmp_dir, os.W_OK),  # read + write
+                "is_valid": os.path.isdir(tmp_dir)
+                and os.access(tmp_dir, os.R_OK)
+                and os.access(tmp_dir, os.W_OK)
+                and tmp_dir_socket_path_is_short_enough(tmp_dir),
             },
-            # "CACHE_DIR": {
-            #     "path": CACHE_DIR.resolve(),
-            #     "enabled": True,
-            #     "is_valid": os.access(CACHE_DIR, os.R_OK) and os.access(CACHE_DIR, os.W_OK),                        # read + write
-            # },
         },
     )
 
@@ -347,11 +373,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
 
     config = config or get_config(**config_kwargs)
     try:
-        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.LIB_DIR
+        lib_dir = get_or_create_working_lib_dir(autofix=True, quiet=True, config=config) or config.ABXPKG_LIB_DIR
     except Exception:
-        lib_dir = config.LIB_DIR
-
-    lib_bin_dir = lib_dir / "bin"
+        lib_dir = config.ABXPKG_LIB_DIR
 
     return AttrDict(
         {
@@ -375,17 +399,10 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "enabled": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR),
                 "is_valid": os.path.isdir(CONSTANTS.USER_PLUGINS_DIR) and os.access(CONSTANTS.USER_PLUGINS_DIR, os.R_OK),  # read
             },
-            "LIB_DIR": {
+            "ABXPKG_LIB_DIR": {
                 "path": lib_dir.resolve(),
                 "enabled": True,
                 "is_valid": os.path.isdir(lib_dir) and os.access(lib_dir, os.R_OK) and os.access(lib_dir, os.W_OK),  # read + write
             },
-            "LIB_BIN_DIR": {
-                "path": lib_bin_dir.resolve(),
-                "enabled": True,
-                "is_valid": os.path.isdir(lib_bin_dir)
-                and os.access(lib_bin_dir, os.R_OK)
-                and os.access(lib_bin_dir, os.W_OK),  # read + write
-            },
         },
     )

archivebox/config/permissions.py

@@ -1,19 +1,53 @@
 __package__ = "archivebox.config"
 
 import os
-import pwd
-import sys
-import socket
 import platform
+import pwd
+import socket
+import stat
+import subprocess
+import sys
+from contextlib import contextmanager, suppress
+from pathlib import Path
 from typing import cast
 
 from rich import print
 
-from pathlib import Path
-from contextlib import contextmanager
-
 #############################################################################################
 
+
+def is_root_identity(running_uid: int, effective_uid: int) -> bool:
+    return running_uid == 0 or effective_uid == 0
+
+
+def select_archivebox_user(
+    *,
+    running_uid: int,
+    running_gid: int,
+    effective_uid: int,
+    effective_gid: int,
+    sudo_uid: int,
+    sudo_gid: int,
+    data_dir_uid: int,
+    data_dir_gid: int,
+    account_uid: int | None,
+    account_gid: int | None,
+    data_dir_owner_gid: int | None = None,
+    data_dir_owner_exists: bool = True,
+) -> tuple[int, int]:
+    if running_uid == 0:
+        if data_dir_uid != 0 and data_dir_owner_exists:
+            owner_gid = data_dir_owner_gid if data_dir_gid == 0 and data_dir_owner_gid is not None else data_dir_gid
+            return data_dir_uid, owner_gid
+        if account_uid is not None and account_gid is not None:
+            return account_uid, account_gid
+
+    return (
+        effective_uid or running_uid or sudo_uid,
+        effective_gid or running_gid or sudo_gid,
+    )
+
+
 DATA_DIR = Path(os.getcwd())
 
 try:
@@ -24,47 +58,100 @@ except PermissionError:
     DATA_DIR_UID = 0
     DATA_DIR_GID = 0
 
-DEFAULT_PUID = 911
-DEFAULT_PGID = 911
+try:
+    DATA_DIR_OWNER_GID = pwd.getpwuid(DATA_DIR_UID).pw_gid
+    DATA_DIR_OWNER_EXISTS = True
+except KeyError:
+    DATA_DIR_OWNER_GID = None
+    DATA_DIR_OWNER_EXISTS = False
+
+DEFAULT_UID = 911
+DEFAULT_GID = 911
 RUNNING_AS_UID = os.getuid()
 RUNNING_AS_GID = os.getgid()
 EUID = os.geteuid()
 EGID = os.getegid()
-SUDO_UID = int(os.environ.get("SUDO_UID", 0))
-SUDO_GID = int(os.environ.get("SUDO_GID", 0))
+SUDO_UID = int(os.environ.get("SUDO_UID", "0"))
+SUDO_GID = int(os.environ.get("SUDO_GID", "0"))
 USER: str = Path("~").expanduser().resolve().name
 HOSTNAME: str = cast(str, max([socket.gethostname(), platform.node()], key=len))
 
-IS_ROOT = RUNNING_AS_UID == 0
-IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
-# IN_DOCKER_COMPOSE =  # TODO: figure out a way to detect if running in docker compose
-
+IS_ROOT = is_root_identity(RUNNING_AS_UID, EUID)
+IN_DOCKER = os.environ.get("IN_DOCKER", "") in ("1", "true", "True", "TRUE", "yes")
 
 FALLBACK_UID = RUNNING_AS_UID or SUDO_UID
 FALLBACK_GID = RUNNING_AS_GID or SUDO_GID
-if RUNNING_AS_UID == 0:
+
+
+def get_or_create_archivebox_account():
     try:
-        # if we are running as root it's really hard to figure out what the correct archivebox user should be
-        # as a last resort instead of setting DATA_DIR ownership to 0:0 (which breaks it for non-root users)
-        # check if 911:911 archivebox user exists on host system, and use it instead of 0
-        if pwd.getpwuid(DEFAULT_PUID).pw_name == "archivebox":
-            FALLBACK_UID = DEFAULT_PUID
-            FALLBACK_GID = DEFAULT_PGID
-    except Exception:
-        pass
+        account = pwd.getpwnam("archivebox")
+    except KeyError:
+        account = None
+
+    if account is None and (RUNNING_AS_UID != 0 or platform.system() != "Linux"):
+        return None
+
+    if account is None:
+        useradd = "/usr/sbin/useradd" if Path("/usr/sbin/useradd").exists() else "useradd"
+        command = [
+            useradd,
+            "--system",
+            "--create-home",
+            "--home-dir",
+            "/var/lib/archivebox",
+            "--shell",
+            "/bin/bash",
+            "archivebox",
+        ]
+        try:
+            subprocess.run(command, check=True)
+        except (OSError, subprocess.CalledProcessError) as err:
+            # Another concurrently starting process may have created it first.
+            try:
+                account = pwd.getpwnam("archivebox")
+            except KeyError:
+                raise RuntimeError(f"Failed to create the archivebox system user with: {' '.join(command)}") from err
+        else:
+            account = pwd.getpwnam("archivebox")
+
+    if RUNNING_AS_UID == 0 and platform.system() == "Linux":
+        # Package removal may preserve the account but remove its home. Repair
+        # only this directory entry; never recursively chown existing data.
+        account_home = Path(account.pw_dir)
+        try:
+            account_home.mkdir(parents=True, exist_ok=True)
+            home_stat = account_home.stat()
+            if (home_stat.st_uid, home_stat.st_gid) != (account.pw_uid, account.pw_gid):
+                os.chown(account_home, account.pw_uid, account.pw_gid)
+        except OSError as err:
+            raise RuntimeError(f"Failed to prepare archivebox account home: {account_home}") from err
+
+    return account
 
 
-os.environ.setdefault("PUID", str(DATA_DIR_UID or EUID or RUNNING_AS_UID or FALLBACK_UID))
-os.environ.setdefault("PGID", str(DATA_DIR_GID or EGID or RUNNING_AS_GID or FALLBACK_GID))
+ARCHIVEBOX_ACCOUNT = get_or_create_archivebox_account()
 
-ARCHIVEBOX_USER = int(os.environ["PUID"])
-ARCHIVEBOX_GROUP = int(os.environ["PGID"])
+ARCHIVEBOX_USER, ARCHIVEBOX_GROUP = select_archivebox_user(
+    running_uid=RUNNING_AS_UID,
+    running_gid=RUNNING_AS_GID,
+    effective_uid=EUID,
+    effective_gid=EGID,
+    sudo_uid=SUDO_UID,
+    sudo_gid=SUDO_GID,
+    data_dir_uid=DATA_DIR_UID,
+    data_dir_gid=DATA_DIR_GID,
+    data_dir_owner_gid=DATA_DIR_OWNER_GID,
+    account_uid=ARCHIVEBOX_ACCOUNT.pw_uid if ARCHIVEBOX_ACCOUNT is not None else None,
+    account_gid=ARCHIVEBOX_ACCOUNT.pw_gid if ARCHIVEBOX_ACCOUNT is not None else None,
+    data_dir_owner_exists=DATA_DIR_OWNER_EXISTS,
+)
 if not USER:
     try:
         # alternative method 1 to get username
         USER = pwd.getpwuid(ARCHIVEBOX_USER).pw_name
-    except Exception:
-        pass
+    except (KeyError, OSError):
+        USER = ""
 
 if not USER:
     try:
@@ -72,42 +159,172 @@ if not USER:
         import getpass
 
         USER = getpass.getuser()
-    except Exception:
-        pass
+    except OSError:
+        USER = ""
 
 if not USER:
     try:
         # alternative method 3 to get username
         USER = os.getlogin() or "archivebox"
-    except Exception:
+    except OSError:
         USER = "archivebox"
 
 ARCHIVEBOX_USER_EXISTS = False
 try:
     pwd.getpwuid(ARCHIVEBOX_USER)
     ARCHIVEBOX_USER_EXISTS = True
-except Exception:
+except KeyError:
     ARCHIVEBOX_USER_EXISTS = False
 
 
+ROOT_HANDOFF_NAMES = (
+    ".archivebox_id",
+    "ArchiveBox.conf",
+    "index.sqlite3",
+    "index.sqlite3-shm",
+    "index.sqlite3-wal",
+    "archive",
+    "cache",
+    "custom_plugins",
+    "custom_templates",
+    "lib",
+    "logs",
+    "personas",
+    "sonic",
+    "sources",
+    "tmp",
+    "users",
+)
+
+
+def root_data_dir_handoff_paths(data_dir: Path, argv: list[str]) -> tuple[Path, ...]:
+    """Return bounded top-level paths safe to hand from root to archivebox."""
+
+    data_dir = data_dir.resolve()
+    if data_dir == Path("/"):
+        return ()
+
+    try:
+        children = tuple(data_dir.iterdir())
+    except (FileNotFoundError, PermissionError):
+        return ()
+
+    is_init = any(arg in ("init", "install", "--init", "--quick-init") for arg in argv[1:])
+    collection_exists = any((data_dir / marker).exists() for marker in (".archivebox_id", "ArchiveBox.conf", "index.sqlite3"))
+    if not collection_exists and not (is_init and not children):
+        return ()
+
+    top_level_paths = tuple(data_dir / name for name in ROOT_HANDOFF_NAMES if (data_dir / name).exists())
+    log_files = tuple((data_dir / "logs").glob("*.log"))
+    return (data_dir, *top_level_paths, *log_files)
+
+
+def root_should_handoff_data_dir(
+    *,
+    is_root: bool,
+    data_dir_uid: int,
+    account_uid: int | None,
+    data_dir_owner_exists: bool = True,
+) -> bool:
+    """Allow bounded handoff only for root or archivebox-owned collection roots."""
+
+    return is_root and account_uid is not None and (not data_dir_owner_exists or data_dir_uid in (0, account_uid))
+
+
+def root_parent_can_grant_group_traversal(*, parent_uid: int, parent_gid: int, parent_mode: int, account_gid: int) -> bool:
+    """Return whether adding archivebox traversal preserves existing parent access."""
+
+    if parent_uid != 0 or parent_mode & stat.S_IXOTH:
+        return False
+    if parent_gid == account_gid:
+        return not bool(parent_mode & stat.S_IXGRP)
+    return not bool(parent_mode & stat.S_IRWXG)
+
+
+def grant_archivebox_group_traversal(path: Path) -> None:
+    """Let the archivebox account traverse private root-owned parents."""
+
+    if not IS_ROOT or ARCHIVEBOX_ACCOUNT is None:
+        return
+
+    for parent in path.resolve().parents:
+        if parent == Path("/"):
+            continue
+        parent_stat = parent.stat(follow_symlinks=False)
+        if root_parent_can_grant_group_traversal(
+            parent_uid=parent_stat.st_uid,
+            parent_gid=parent_stat.st_gid,
+            parent_mode=parent_stat.st_mode,
+            account_gid=ARCHIVEBOX_ACCOUNT.pw_gid,
+        ):
+            os.chown(parent, -1, ARCHIVEBOX_ACCOUNT.pw_gid, follow_symlinks=False)
+            os.chmod(parent, stat.S_IMODE(parent_stat.st_mode) | stat.S_IXGRP, follow_symlinks=False)
+
+
+def handoff_root_owned_data_dir() -> None:
+    account_uid = ARCHIVEBOX_ACCOUNT.pw_uid if ARCHIVEBOX_ACCOUNT is not None else None
+    if not root_should_handoff_data_dir(
+        is_root=IS_ROOT,
+        data_dir_uid=DATA_DIR_UID,
+        account_uid=account_uid,
+        data_dir_owner_exists=DATA_DIR_OWNER_EXISTS,
+    ):
+        return
+
+    handoff_paths = root_data_dir_handoff_paths(DATA_DIR, sys.argv)
+    if not handoff_paths:
+        return
+
+    # Never walk collection contents. This only grants execute-only traversal
+    # on private parents such as /root so the handed-off data remains reachable.
+    grant_archivebox_group_traversal(DATA_DIR)
+
+    for path in handoff_paths:
+        try:
+            os.chown(path, ARCHIVEBOX_ACCOUNT.pw_uid, ARCHIVEBOX_ACCOUNT.pw_gid, follow_symlinks=False)
+        except (FileNotFoundError, PermissionError):
+            pass
+
+
 #############################################################################################
 
 
 def drop_privileges():
-    """If running as root, drop privileges to the user that owns the data dir (or PUID)"""
+    """If running as root, drop privileges to the data dir owner or archivebox user."""
 
-    # always run archivebox as the user that owns the data dir, never as root
-    if os.getuid() == 0:
-        # drop permissions to the user that owns the data dir / provided PUID
-        if os.geteuid() != ARCHIVEBOX_USER and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
-            # drop our effective UID to the archivebox user's UID
+    # Root-owned uv tools commonly live below /root. Keep the installed package
+    # importable after dropping EUID without changing ownership of the tool env.
+    grant_archivebox_group_traversal(Path(__file__))
+    handoff_root_owned_data_dir()
+
+    # Always run ArchiveBox as the user that owns the data dir, or as the
+    # archivebox service account when the data dir is root-owned.
+    if os.geteuid() == 0 and ARCHIVEBOX_USER != 0 and ARCHIVEBOX_USER_EXISTS:
+        pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
+        supplemental_gid = ARCHIVEBOX_ACCOUNT.pw_gid if ARCHIVEBOX_ACCOUNT is not None else ARCHIVEBOX_GROUP
+        os.initgroups(pw_record.pw_name, supplemental_gid)
+        if os.getegid() != ARCHIVEBOX_GROUP:
+            os.setegid(ARCHIVEBOX_GROUP)
+        if os.geteuid() != ARCHIVEBOX_USER:
             os.seteuid(ARCHIVEBOX_USER)
 
-            # update environment variables so that subprocesses dont try to write to /root
-            pw_record = pwd.getpwuid(ARCHIVEBOX_USER)
-            os.environ["HOME"] = pw_record.pw_dir
-            os.environ["LOGNAME"] = pw_record.pw_name
-            os.environ["USER"] = pw_record.pw_name
+        # update environment variables so that subprocesses dont try to write to /root
+        os.environ["HOME"] = pw_record.pw_dir
+        os.environ["LOGNAME"] = pw_record.pw_name
+        os.environ["USER"] = pw_record.pw_name
+        os.environ["XDG_CACHE_HOME"] = str(Path(pw_record.pw_dir) / ".cache")
+        os.environ["XDG_CONFIG_HOME"] = str(Path(pw_record.pw_dir) / ".config")
+        os.environ["XDG_DATA_HOME"] = str(Path(pw_record.pw_dir) / ".local" / "share")
+        os.environ.pop("XDG_RUNTIME_DIR", None)
+        os.environ.pop("ABXBUS_MULTIPROCESS_SEMAPHORE_DIR", None)
+
+        semaphore_dir = Path(pw_record.pw_dir) / ".cache" / "abxbus" / "semaphores"
+        os.environ["ABXBUS_MULTIPROCESS_SEMAPHORE_DIR"] = str(semaphore_dir)
+
+        with suppress(ImportError):
+            from abxbus import retry as abxbus_retry
+
+            abxbus_retry.MULTIPROCESS_SEMAPHORE_DIR = semaphore_dir
 
     if ARCHIVEBOX_USER == 0 or not ARCHIVEBOX_USER_EXISTS:
         print(

archivebox/config/version.py

@@ -36,26 +36,32 @@ def detect_installed_version(PACKAGE_DIR: Path = PACKAGE_DIR):
         # building docs, pyproject.toml is not available
         pass
 
-    # raise Exception('Failed to detect installed archivebox version!')
     return "dev"
 
 
 @cache
 def get_COMMIT_HASH() -> str | None:
-    for env_var in ("ARCHIVEBOX_COMMIT_HASH", "COMMIT_HASH"):
+    for env_var in ("ARCHIVEBOX_COMMIT_HASH",):
         env_commit_hash = os.environ.get(env_var, "").strip()
         if re.fullmatch(r"[0-9a-fA-F]{40}", env_commit_hash):
             return env_commit_hash
 
     if IN_DOCKER:
         try:
-            version_txt = Path("/VERSION.txt").read_text()
-            docker_commit_hashes = re.findall(r"COMMIT_HASH=([0-9a-fA-F]{40})", version_txt)
-            if docker_commit_hashes:
-                return docker_commit_hashes[-1]
+            version_text = Path("/VERSION.txt").read_text()
+            matches = re.findall(r"^COMMIT_HASH=([0-9a-fA-F]{40})$", version_text, re.MULTILINE)
+            if matches:
+                return matches[-1]
         except Exception:
             pass
 
+    try:
+        packaged_commit_hash = (PACKAGE_DIR / "COMMIT_SHA").read_text().strip()
+        if re.fullmatch(r"[0-9a-fA-F]{40}", packaged_commit_hash):
+            return packaged_commit_hash
+    except Exception:
+        pass
+
     def _read_git_file(git_dir: Path, ref: str) -> str | None:
         try:
             return git_dir.joinpath(ref).read_text().strip()
@@ -77,6 +83,13 @@ def get_COMMIT_HASH() -> str | None:
         return None
 
     try:
+        try:
+            pyproject_text = (PACKAGE_DIR.parent / "pyproject.toml").read_text()
+        except FileNotFoundError:
+            pyproject_text = ""
+        if not re.search(r'^name = "archivebox"$', pyproject_text, re.MULTILINE):
+            return None
+
         git_dir = PACKAGE_DIR.parent / ".git"
         if git_dir.is_file():
             gitdir_line = git_dir.read_text().strip()
@@ -96,75 +109,18 @@ def get_COMMIT_HASH() -> str | None:
     except Exception:
         pass
 
-    try:
-        return list((PACKAGE_DIR.parent / ".git/refs/heads/").glob("*"))[0].read_text().strip()
-    except Exception:
-        pass
-
     return None
 
 
 @cache
 def get_BUILD_TIME() -> str:
-    if IN_DOCKER:
-        try:
-            # if we're in the archivebox official docker image, /VERSION.txt will contain the build time
-            docker_build_end_time = Path("/VERSION.txt").read_text().rsplit("BUILD_END_TIME=")[-1].split("\n", 1)[0]
-            return docker_build_end_time
-        except Exception:
-            pass
+    for env_var in ("ARCHIVEBOX_BUILD_TIME", "BUILD_TIME"):
+        build_time = os.environ.get(env_var, "").strip()
+        if build_time:
+            return build_time
 
     src_last_modified_unix_timestamp = (PACKAGE_DIR / "README.md").stat().st_mtime
     return datetime.fromtimestamp(src_last_modified_unix_timestamp).strftime("%Y-%m-%d %H:%M:%S %s")
 
 
-# def get_versions_available_on_github(config):
-#     """
-#     returns a dictionary containing the ArchiveBox GitHub release info for
-#     the recommended upgrade version and the currently installed version
-#     """
-
-#     # we only want to perform the (relatively expensive) check for new versions
-#     # when its most relevant, e.g. when the user runs a long-running command
-#     subcommand_run_by_user = sys.argv[3] if len(sys.argv) > 3 else 'help'
-#     long_running_commands = ('add', 'schedule', 'update', 'status', 'server')
-#     if subcommand_run_by_user not in long_running_commands:
-#         return None
-
-#     github_releases_api = "https://api.github.com/repos/ArchiveBox/ArchiveBox/releases"
-#     response = requests.get(github_releases_api)
-#     if response.status_code != 200:
-#         stderr(f'[!] Warning: GitHub API call to check for new ArchiveBox version failed! (status={response.status_code})', color='lightyellow', config=config)
-#         return None
-#     all_releases = response.json()
-
-#     installed_version = parse_version_string(config['VERSION'])
-
-#     # find current version or nearest older version (to link to)
-#     current_version = None
-#     for idx, release in enumerate(all_releases):
-#         release_version = parse_version_string(release['tag_name'])
-#         if release_version <= installed_version:
-#             current_version = release
-#             break
-
-#     current_version = current_version or all_releases[-1]
-
-#     # recommended version is whatever comes after current_version in the release list
-#     # (perhaps too conservative to only recommend upgrading one version at a time, but it's safest)
-#     try:
-#         recommended_version = all_releases[idx+1]
-#     except IndexError:
-#         recommended_version = None
-
-#     return {'recommended_version': recommended_version, 'current_version': current_version}
-
-# def can_upgrade(config):
-#     if config['VERSIONS_AVAILABLE'] and config['VERSIONS_AVAILABLE']['recommended_version']:
-#         recommended_version = parse_version_string(config['VERSIONS_AVAILABLE']['recommended_version']['tag_name'])
-#         current_version = parse_version_string(config['VERSIONS_AVAILABLE']['current_version']['tag_name'])
-#         return recommended_version > current_version
-#     return False
-
-
 VERSION: str = detect_installed_version()

archivebox/config/views.py

@@ -1,26 +1,32 @@
 __package__ = "archivebox.config"
 
-import os
 import inspect
+import os
 from pathlib import Path
 from typing import Any
 from urllib.parse import quote, urlencode
+
+from admin_data_views.typing import ItemContext, SectionData, TableContext
+from admin_data_views.utils import ItemLink, render_with_item_view, render_with_table_view
 from django.http import HttpRequest
 from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 
-from admin_data_views.typing import TableContext, ItemContext, SectionData
-from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
-
 from archivebox.config import CONSTANTS
+from archivebox.machine.models import Binary
 from archivebox.misc.util import parse_date
 
-from archivebox.machine.models import Binary
-
-LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 ENVIRONMENT_BINARIES_BASE_URL = "/admin/environment/binaries/"
 INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
+LOG_DETAIL_TAIL_BYTES = 100_000
+
+
+def _read_text_tail(path: Path, max_bytes: int = LOG_DETAIL_TAIL_BYTES) -> str:
+    with path.open("rb") as log_file:
+        size = path.stat().st_size
+        log_file.seek(max(size - max_bytes, 0))
+        return log_file.read(max_bytes).decode("utf-8", errors="replace")
 
 
 def is_superuser(request: HttpRequest) -> bool:
@@ -32,10 +38,6 @@ def format_parsed_datetime(value: object) -> str:
     return parsed.strftime("%Y-%m-%d %H:%M:%S") if parsed else ""
 
 
-def get_live_config_url(key: str) -> str:
-    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
-
-
 def get_environment_binary_url(name: str) -> str:
     return f"{ENVIRONMENT_BINARIES_BASE_URL}{quote(name)}/"
 
@@ -92,7 +94,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
             return f" {obj}"
 
     elif isinstance(obj, (int, float, bool)):
-        return f" {str(obj)}"
+        return f" {obj!s}"
 
     elif callable(obj):
         source = (
@@ -103,7 +105,7 @@ def obj_to_yaml(obj: Any, indent: int = 0) -> str:
         return f" {indent_str}  " + source.replace("\n", f"\n{indent_str}  ")
 
     else:
-        return f" {str(obj)}"
+        return f" {obj!s}"
 
 
 def _binary_sort_key(binary: Binary) -> tuple[int, int, int, Any]:
@@ -305,7 +307,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 pid_to_process_id[row.pid] = str(row.id)
                 pid_to_process_type[row.pid] = row.process_type
 
-        except Exception:
+        except (ImportError, RuntimeError, TypeError, ValueError):
             pass
 
     def _pid_cell(pid_value: int | None, uptime_str: str = ""):
@@ -346,7 +348,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
         hours, remainder = divmod(seconds, 3600)
         minutes, secs = divmod(remainder, 60)
         supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
-    except Exception:
+    except (ImportError, RuntimeError, TypeError, ValueError):
         try:
             from archivebox.machine.models import Machine, Process
 
@@ -365,7 +367,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
                 hours, remainder = divmod(seconds, 3600)
                 minutes, secs = divmod(remainder, 60)
                 supervisor_uptime = f"{hours}:{minutes:02d}:{secs:02d}"
-        except Exception:
+        except (RuntimeError, TypeError, ValueError):
             pass
 
     rows["PID"].append(_pid_cell(supervisor_pid if isinstance(supervisor_pid, int) else None, supervisor_uptime))
@@ -420,7 +422,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
 def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
-    from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME
+    from archivebox.workers.supervisord_util import CONFIG_FILE_NAME, get_existing_supervisord_process, get_sock_file, get_worker
 
     SOCK_FILE = get_sock_file()
     CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME
@@ -443,8 +445,12 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
     if key == "supervisord":
         relevant_config = CONFIG_FILE.read_text()
-        relevant_logs = str(supervisor.readLog(0, 10_000_000))
-        start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
+        supervisor_log = CONSTANTS.LOGS_DIR / "supervisord.log"
+        with supervisor_log.open("rb") as log_file:
+            startup_logs = log_file.read(LOG_DETAIL_TAIL_BYTES).decode("utf-8", errors="replace")
+        relevant_logs = _read_text_tail(supervisor_log)
+        startup_lines = [line for line in startup_logs.split("\n") if "RPC interface 'supervisor' initialized" in line]
+        start_ts = startup_lines[-1].split(",", 1)[0] if startup_lines else ""
         start_dt = parse_date(start_ts)
         uptime = str(timezone.now() - start_dt).split(".")[0] if start_dt else ""
         supervisor_state = supervisor.getState()
@@ -463,7 +469,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         worker_data = get_worker(supervisor, key)
         proc = worker_data if isinstance(worker_data, dict) else {}
         relevant_config = next((config for config in all_config if config.get("name") == key), {})
-        log_result = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)
+        log_result = supervisor.tailProcessStdoutLog(key, -LOG_DETAIL_TAIL_BYTES, LOG_DETAIL_TAIL_BYTES)
         relevant_logs = str(log_result[0] if isinstance(log_result, tuple) else log_result)
 
     section: SectionData = {
@@ -479,7 +485,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
             "Logfile": str(proc.get("stdout_logfile") or ""),
             "Uptime": str(str(proc.get("description") or "").split("uptime ", 1)[-1]),
             "Config": obj_to_yaml(relevant_config) if isinstance(relevant_config, dict) else str(relevant_config),
-            "Logs": relevant_logs,
+            "Recent Logs (last 100 KB)": relevant_logs,
         },
         "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
     }
@@ -533,9 +539,9 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
 def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     assert is_superuser(request), "Must be a superuser to view configuration settings."
 
-    log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name][0]
+    log_file = next(logfile for logfile in CONSTANTS.LOGS_DIR.glob("*.log") if key in logfile.name)
 
-    log_text = log_file.read_text()
+    log_text = _read_text_tail(log_file)
     log_stat = log_file.stat()
 
     section: SectionData = {
@@ -545,8 +551,8 @@ def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
             "Path": str(log_file),
             "Size": f"{log_stat.st_size // 1000} kb",
             "Last Updated": format_parsed_datetime(log_stat.st_mtime),
-            "Tail": "\n".join(log_text[-10_000:].split("\n")[-20:]),
-            "Full Log": log_text,
+            "Tail": "\n".join(log_text.split("\n")[-20:]),
+            "Recent Log (last 100 KB)": log_text,
         },
     }
 

archivebox/core/admin_archiveresults.py

@@ -4,31 +4,29 @@ import html
 import json
 import os
 import shlex
-from pathlib import Path
-from urllib.parse import quote
 from functools import reduce
 from operator import and_
+from pathlib import Path
+from urllib.parse import quote
 
 from django.contrib import admin
+from django.core.exceptions import ValidationError
 from django.db.models import Min, Prefetch, Q, TextField
 from django.db.models.functions import Cast
+from django.urls import resolve, reverse
+from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
-from django.core.exceptions import ValidationError
-from django.urls import reverse, resolve
-from django.utils import timezone
 from django.utils.text import smart_split
 
-from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
-from archivebox.plugins.discovery import get_plugin_icon
-from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
+from archivebox.core.models import ArchiveResult, Snapshot
 from archivebox.core.routes_util import build_snapshot_url
 from archivebox.core.widgets import InlineTagEditorWidget
 from archivebox.machine.env_util import env_to_shell_exports
-
-
-from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.misc.paginators import AcceleratedPaginator
+from archivebox.plugins.discovery import get_plugin_icon
+from archivebox.plugins.views import LIVE_PLUGIN_BASE_URL
 
 
 def _get_replay_source_url(result: ArchiveResult) -> str:
@@ -120,15 +118,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         status = result.status or "queued"
         color, bg = status_colors.get(status, ("#6b7280", "#f3f4f6"))
         output_files = result.output_files or {}
-        if isinstance(output_files, dict):
-            output_file_count = len(output_files)
-        elif isinstance(output_files, (list, tuple, set)):
+        if isinstance(output_files, (dict, list, tuple, set)):
             output_file_count = len(output_files)
         elif isinstance(output_files, str):
             try:
                 parsed = json.loads(output_files)
                 output_file_count = len(parsed) if isinstance(parsed, (dict, list, tuple, set)) else 0
-            except Exception:
+            except (TypeError, ValueError):
                 output_file_count = 0
         else:
             output_file_count = 0
@@ -136,31 +132,43 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         # Get plugin icon
         icon = get_plugin_icon(result.plugin)
 
-        # Format timestamp
-        end_time = result.end_ts.strftime("%Y-%m-%d %H:%M:%S") if result.end_ts else "-"
+        # Keep each timestamp component intact while allowing one wrap between them.
+        if result.end_ts:
+            end_time = (
+                f'<span style="white-space: nowrap;">{result.end_ts:%Y-%m-%d}</span>'
+                f'<wbr> <span style="white-space: nowrap;">{result.end_ts:%H:%M:%S}</span>'
+            )
+        else:
+            end_time = "-"
 
         process = result.process_record
         process_display = "-"
         if process:
+            process_url = html.escape(reverse("admin:machine_process_change", args=[process.id]), quote=True)
+            process_label = html.escape(get_process_link_label(process), quote=True)
             process_display = f'''
-                <a href="{reverse("admin:machine_process_change", args=[process.id])}"
+                <a href="{process_url}"
                    style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px;"
-                   title="View process">{get_process_link_label(process)}</a>
+                   title="View process">{process_label}</a>
             '''
 
         machine_display = "-"
         if process and process.machine_id:
+            machine_url = html.escape(reverse("admin:machine_machine_change", args=[process.machine_id]), quote=True)
+            machine_label = html.escape(str(process.machine.hostname or ""), quote=True)
             machine_display = f'''
-                <a href="{reverse("admin:machine_machine_change", args=[process.machine_id])}"
+                <a href="{machine_url}"
                    style="color: #2563eb; text-decoration: none; font-size: 12px;"
-                   title="View machine">{process.machine.hostname}</a>
+                   title="View machine">{machine_label}</a>
             '''
 
         # Truncate output for display
-        full_output = result.output_str_for_display() or "-"
-        output_display = full_output[:60]
-        if len(full_output) > 60:
-            output_display += "..."
+        full_output_raw = result.output_str_for_display() or "-"
+        output_display_raw = full_output_raw[:60]
+        if len(full_output_raw) > 60:
+            output_display_raw += "..."
+        full_output = html.escape(full_output_raw)
+        output_display = html.escape(output_display_raw)
 
         display_cmd = build_abx_dl_display_command(result)
         replay_cmd = build_abx_dl_replay_command(result, config=config)
@@ -174,9 +182,13 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
             output_link = build_snapshot_url(snapshot_id, "", config=config)
+        output_link_attr = html.escape(output_link, quote=True)
 
         # Get version - try cmd_version field
-        version = result.cmd_version if result.cmd_version else "-"
+        version = html.escape(str(result.cmd_version if result.cmd_version else "-"), quote=True)
+        plugin_text = html.escape(str(result.plugin or ""), quote=True)
+        status_text = html.escape(str(status), quote=True)
+        pwd_text = html.escape(str(result.pwd or "-"), quote=True)
 
         # Unique ID for this row's expandable output
         row_id = f"output_{idx}_{str(result.id)[:8]}"
@@ -193,31 +205,32 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 <td style="padding: 10px 12px; white-space: nowrap;">
                     <span style="display: inline-block; padding: 3px 10px; border-radius: 12px;
                                  font-size: 11px; font-weight: 600; text-transform: uppercase;
-                                 color: {color}; background: {bg};">{status}</span>
+                                 color: {color}; background: {bg};">{status_text}</span>
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{result.plugin}">
+                <td style="padding: 10px 12px; white-space: nowrap; font-size: 20px;" title="{plugin_text}">
                     {icon}
                 </td>
-                <td style="padding: 10px 12px; font-weight: 500; color: #334155;">
-                        <a href="{output_link}" target="_blank"
+                <td class="archive-results-plugin" style="padding: 10px 12px; font-weight: 500; color: #334155;">
+                        <a href="{output_link_attr}" target="_blank"
                            style="color: #334155; text-decoration: none;"
                        title="View output fullscreen"
                        onmouseover="this.style.color='#2563eb'; this.style.textDecoration='underline';"
                        onmouseout="this.style.color='#334155'; this.style.textDecoration='none';">
-                        {result.plugin}
+                        {plugin_text}
                     </a>
                 </td>
-                <td style="padding: 10px 12px; max-width: 280px;">
+                <td class="archive-results-output" style="padding: 10px 12px; max-width: 280px;">
                     <span onclick="document.getElementById('{row_id}').open = !document.getElementById('{row_id}').open"
-                          style="color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
+                          style="display: block; max-width: 280px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
+                                 color: #2563eb; text-decoration: none; font-family: ui-monospace, monospace; font-size: 12px; cursor: pointer;"
                           title="Click to expand full output">
                         {output_display}
                     </span>
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; color: #64748b; font-size: 12px; text-align: right;">
+                <td class="archive-results-files" style="padding: 10px 12px; color: #64748b; font-size: 12px; text-align: right;">
                     {output_file_count}
                 </td>
-                <td style="padding: 10px 12px; white-space: nowrap; color: #64748b; font-size: 12px;">
+                <td class="archive-results-completed" style="padding: 10px 12px; color: #64748b; font-size: 12px;">
                     {end_time}
                 </td>
                 <td style="padding: 10px 12px; white-space: nowrap;">
@@ -231,7 +244,7 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                 </td>
                 <td style="padding: 10px 8px; white-space: nowrap;">
                     <div style="display: flex; gap: 4px;">
-                        <a href="{output_link}" target="_blank"
+                        <a href="{output_link_attr}" target="_blank"
                            style="padding: 4px 8px; background: #f1f5f9; border-radius: 4px; color: #475569; text-decoration: none; font-size: 11px;"
                            title="View output">📄</a>
                         <a href="{reverse("admin:core_archiveresult_change", args=[result.id])}"
@@ -248,9 +261,9 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
                         </summary>
                         <div style="margin-top: 8px; padding: 10px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 6px; max-height: 200px; overflow: auto;">
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
-                                <span style="margin-right: 16px;"><b>ID:</b> <code>{str(result.id)}</code></span>
+                                <span style="margin-right: 16px;"><b>ID:</b> <code>{result.id!s}</code></span>
                                 <span style="margin-right: 16px;"><b>Version:</b> <code>{version}</code></span>
-                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{result.pwd or "-"}</code></span>
+                                <span style="margin-right: 16px;"><b>PWD:</b> <code>{pwd_text}</code></span>
                             </div>
                             <div style="font-size: 11px; color: #64748b; margin-bottom: 8px;">
                                 <b>Output:</b>
@@ -288,17 +301,17 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         """
 
     return mark_safe(f"""
-        <div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow: hidden; background: #fff; width: 100%;">
-            <table style="width: 100%; border-collapse: collapse; font-size: 14px;">
+        <div style="border: 1px solid #e2e8f0; border-radius: 8px; overflow-x: auto; overflow-y: hidden; background: #fff; width: 100%;">
+            <table class="archive-results-table" style="width: 100%; min-width: 1100px; border-collapse: collapse; font-size: 14px;">
                 <thead>
                     <tr style="background: #f8fafc; border-bottom: 2px solid #e2e8f0;">
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Details</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Status</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; width: 32px;"></th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
-                        <th style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
-                        <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
+                        <th class="archive-results-plugin" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Plugin</th>
+                        <th class="archive-results-output" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Output</th>
+                        <th class="archive-results-files" style="padding: 10px 12px; text-align: right; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Files</th>
+                        <th class="archive-results-completed" style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Completed</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Process</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Machine</th>
                         <th style="padding: 10px 12px; text-align: left; font-weight: 600; color: #475569; font-size: 12px; text-transform: uppercase; letter-spacing: 0.05em;">Version</th>
@@ -318,15 +331,12 @@ class ArchiveResultInline(admin.TabularInline):
     name = "Archive Results Log"
     model = ArchiveResult
     parent_model = Snapshot
-    # fk_name = 'snapshot'
     extra = 0
     sort_fields = ("end_ts", "plugin", "output_str", "status", "cmd_version")
     readonly_fields = ("id", "result_id", "completed", "command", "version")
     fields = ("start_ts", "end_ts", *readonly_fields, "plugin", "cmd", "cmd_version", "pwd", "status", "output_str")
-    # exclude = ('id',)
     ordering = ("end_ts",)
     show_change_link = True
-    # # classes = ['collapse']
 
     def get_parent_object_from_request(self, request):
         resolved = resolve(request.path_info)
@@ -361,7 +371,6 @@ class ArchiveResultInline(admin.TabularInline):
         base_fields = formset.form.base_fields
         snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
 
-        # import ipdb; ipdb.set_trace()
         # formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
 
         # default values for new entries
@@ -407,11 +416,11 @@ class ArchiveResultAdmin(BaseModelAdmin):
     list_display_links = None
     sort_fields = ("id", "created_at", "plugin", "status")
     readonly_fields = (
-        "admin_actions",
         "cmd",
         "cmd_version",
         "pwd",
         "cmd_str",
+        "admin_actions",
         "snapshot_info",
         "tags_str",
         "created_at",
@@ -431,20 +440,13 @@ class ArchiveResultAdmin(BaseModelAdmin):
         "output_json",
         "process__cmd",
     )
-    autocomplete_fields = ["snapshot"]
+    autocomplete_fields = ("snapshot",)
 
     fieldsets = (
-        (
-            "Actions",
-            {
-                "fields": ("admin_actions",),
-                "classes": ("card", "wide"),
-            },
-        ),
         (
             "Snapshot",
             {
-                "fields": ("snapshot", "snapshot_info", "tags_str"),
+                "fields": ("snapshot", "snapshot_info", "tags_str", "admin_actions"),
                 "classes": ("card", "wide"),
             },
         ),
@@ -479,14 +481,14 @@ class ArchiveResultAdmin(BaseModelAdmin):
     )
 
     list_filter = ("status", "plugin", "start_ts")
-    ordering = ["-start_ts"]
+    ordering = ("-start_ts",)
     list_per_page = 50
 
     paginator = AcceleratedPaginator
     save_on_top = True
     show_full_result_count = False
 
-    actions = ["delete_selected"]
+    actions = ("delete_selected",)
 
     class Meta:
         verbose_name = "Archive Result"
@@ -496,6 +498,36 @@ class ArchiveResultAdmin(BaseModelAdmin):
         self.request = request
         return super().change_view(request, object_id, form_url, extra_context)
 
+    def get_admin_toolbar_actions(self, request, obj):
+        if obj is None:
+            return []
+        self.request = request
+        return [
+            {
+                "label": "View Output",
+                "icon": "📄",
+                "url": self.get_output_view_url(obj),
+                "title": "Open the archived output for this result",
+            },
+            {
+                "label": "Output files",
+                "icon": "📁",
+                "url": self.get_output_files_url(obj),
+                "title": "Browse the output files for this result",
+            },
+            {
+                "label": "Download Zip",
+                "icon": "⬇",
+                "url": self.get_output_zip_url(obj),
+                "kind": "accent",
+                "css_classes": "archivebox-zip-button",
+                "onclick": "return window.archiveboxHandleZipClick(this, event);",
+                "extra_attrs": [("data-loading-label", "Preparing...")],
+                "title": "Download all output files as a zip",
+            },
+            {"label": "Snapshot", "icon": "🗂", "url": self.get_snapshot_view_url(obj), "title": "Open the parent snapshot view"},
+        ]
+
     def changelist_view(self, request, extra_context=None):
         self.request = request
         saved_list_per_page = self.list_per_page
@@ -606,6 +638,38 @@ class ArchiveResultAdmin(BaseModelAdmin):
             self.get_output_zip_url(result),
         )
 
+    @admin.display(description="")
+    def admin_actions(self, result):
+        return format_html(
+            """
+            <div style="display: flex; flex-wrap: wrap; gap: 12px; align-items: center;">
+                <a class="btn" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; color: #334155; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}">
+                    📄 View Output
+                </a>
+                <a class="btn" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; color: #334155; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}">
+                    📁 Output files
+                </a>
+                <a class="btn archivebox-zip-button" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #eff6ff; border: 1px solid #bfdbfe; border-radius: 8px; color: #1d4ed8; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}"
+                   data-loading-label="Preparing..."
+                   onclick="return window.archiveboxHandleZipClick(this, event);">
+                    <span class="archivebox-zip-spinner" aria-hidden="true"></span>
+                    <span class="archivebox-zip-label">⬇ Download Zip</span>
+                </a>
+                <a class="btn" style="display: inline-flex; align-items: center; gap: 6px; padding: 10px 16px; background: #f8fafc; border: 1px solid #e2e8f0; border-radius: 8px; color: #334155; text-decoration: none; font-size: 14px; font-weight: 500;"
+                   href="{}">
+                    🗂 Snapshot
+                </a>
+            </div>
+            """,
+            self.get_output_view_url(result),
+            self.get_output_files_url(result),
+            self.get_output_zip_url(result),
+            self.get_snapshot_view_url(result),
+        )
+
     @admin.display(
         description="Snapshot",
         ordering="snapshot__url",
@@ -743,46 +807,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
             output_text,
         )
 
-    @admin.display(description="")
-    def admin_actions(self, result):
-        return format_html(
-            """
-            <div style="display:flex; flex-wrap:wrap; gap:12px; align-items:center;">
-                <a class="btn" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#f8fafc; border:1px solid #e2e8f0; border-radius:8px; color:#334155; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   onmouseover="this.style.background='#f1f5f9'; this.style.borderColor='#cbd5e1';"
-                   onmouseout="this.style.background='#f8fafc'; this.style.borderColor='#e2e8f0';">
-                    📄 View Output
-                </a>
-                <a class="btn" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#f8fafc; border:1px solid #e2e8f0; border-radius:8px; color:#334155; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   onmouseover="this.style.background='#f1f5f9'; this.style.borderColor='#cbd5e1';"
-                   onmouseout="this.style.background='#f8fafc'; this.style.borderColor='#e2e8f0';">
-                    📁 Output files
-                </a>
-                <a class="btn archivebox-zip-button" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#eff6ff; border:1px solid #bfdbfe; border-radius:8px; color:#1d4ed8; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   data-loading-label="Preparing..."
-                   onclick="return window.archiveboxHandleZipClick(this, event);"
-                   onmouseover="this.style.background='#dbeafe'; this.style.borderColor='#93c5fd';"
-                   onmouseout="this.style.background='#eff6ff'; this.style.borderColor='#bfdbfe';">
-                    <span class="archivebox-zip-spinner" aria-hidden="true"></span>
-                    <span class="archivebox-zip-label">⬇ Download Zip</span>
-                </a>
-                <a class="btn" style="display:inline-flex; align-items:center; gap:6px; padding:10px 16px; background:#f8fafc; border:1px solid #e2e8f0; border-radius:8px; color:#334155; text-decoration:none; font-size:14px; font-weight:500; transition:all 0.15s;"
-                   href="{}"
-                   onmouseover="this.style.background='#f1f5f9'; this.style.borderColor='#cbd5e1';"
-                   onmouseout="this.style.background='#f8fafc'; this.style.borderColor='#e2e8f0';">
-                    🗂 Snapshot
-                </a>
-            </div>
-            """,
-            self.get_output_view_url(result),
-            self.get_output_files_url(result),
-            self.get_output_zip_url(result),
-            self.get_snapshot_view_url(result),
-        )
-
     def output_summary(self, result):
         snapshot_dir = Path(result.snapshot.output_dir)
         output_html = format_html(
@@ -807,8 +831,6 @@ class ArchiveResultAdmin(BaseModelAdmin):
         else:
             root_dir = str(snapshot_dir)
 
-        # print(root_dir, str(list(os.walk(root_dir))))
-
         for root, dirs, files in os.walk(root_dir):
             depth = root.replace(root_dir, "").count(os.sep) + 1
             if depth > 2:

archivebox/core/admin_site.py

@@ -2,23 +2,46 @@ __package__ = "archivebox.core"
 
 from typing import TYPE_CHECKING, Any
 
-from django.contrib import admin
-from django.db import DatabaseError, connection
 from admin_data_views.admin import (
     admin_data_index_view as adv_admin_data_index_view,
+)
+from admin_data_views.admin import (
     get_admin_data_urls as adv_get_admin_data_urls,
+)
+from admin_data_views.admin import (
     get_app_list as adv_get_app_list,
 )
+from django.contrib import admin
+from django.contrib.auth import REDIRECT_FIELD_NAME
+from django.contrib.auth.decorators import login_not_required
+from django.contrib.auth.views import LoginView
+from django.db import DatabaseError, connection
+from django.http import HttpResponseRedirect
+from django.urls import reverse
+from django.utils.decorators import method_decorator
+from django.utils.translation import gettext as _
+from django.views.decorators.cache import never_cache
 
 from archivebox.config import VERSION
-from archivebox.config.version import get_COMMIT_HASH
+from archivebox.core.context_processors import get_static_cache_key
+from archivebox.core.routes_util import is_allowed_archivebox_redirect_url
 
 if TYPE_CHECKING:
+    from admin_data_views.typing import AppDict
     from django.http import HttpRequest
     from django.template.response import TemplateResponse
     from django.urls import URLPattern, URLResolver
 
-    from admin_data_views.typing import AppDict
+
+class ArchiveBoxLoginView(LoginView):
+    def get_redirect_url(self) -> str:
+        redirect_to = self.request.POST.get(
+            self.redirect_field_name,
+            self.request.GET.get(self.redirect_field_name),
+        )
+        if is_allowed_archivebox_redirect_url(redirect_to, request=self.request):
+            return redirect_to
+        return ""
 
 
 class ArchiveBoxAdmin(admin.AdminSite):
@@ -30,7 +53,7 @@ class ArchiveBoxAdmin(admin.AdminSite):
     def each_context(self, request: "HttpRequest") -> dict[str, Any]:
         context = super().each_context(request)
         context["VERSION"] = VERSION
-        context["STATIC_CACHE_KEY"] = (get_COMMIT_HASH() or VERSION or "dev").strip()
+        context["STATIC_CACHE_KEY"] = get_static_cache_key()
         return context
 
     @staticmethod
@@ -72,10 +95,36 @@ class ArchiveBoxAdmin(admin.AdminSite):
     def admin_data_index_view(self, request: "HttpRequest", **kwargs: Any) -> "TemplateResponse":
         return adv_admin_data_index_view(self, request, **kwargs)
 
+    @method_decorator(never_cache)
+    @login_not_required
+    def login(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
+        if request.method == "GET" and self.has_permission(request):
+            return HttpResponseRedirect(reverse("admin:index", current_app=self.name))
+
+        from django.contrib.admin.forms import AdminAuthenticationForm
+
+        context = {
+            **self.each_context(request),
+            "title": _("Log in"),
+            "subtitle": None,
+            "app_path": request.get_full_path(),
+            "username": request.user.get_username(),
+        }
+        if REDIRECT_FIELD_NAME not in request.GET and REDIRECT_FIELD_NAME not in request.POST:
+            context[REDIRECT_FIELD_NAME] = reverse("admin:index", current_app=self.name)
+        context.update(extra_context or {})
+
+        index_path = reverse("admin:index", current_app=self.name)
+        request.current_app = self.name
+        return ArchiveBoxLoginView.as_view(
+            extra_context=context,
+            authentication_form=self.login_form or AdminAuthenticationForm,
+            template_name=self.login_template or "admin/login.html",
+            next_page=index_path,
+        )(request)
+
     def index(self, request: "HttpRequest", extra_context: dict[str, Any] | None = None) -> "TemplateResponse":
         response = super().index(request, extra_context)
-        if connection.vendor != "sqlite":
-            return response
 
         models_by_table: dict[str, list[dict[str, Any]]] = {}
         for app in response.context_data.get("app_list", []):
@@ -88,23 +137,18 @@ class ArchiveBoxAdmin(admin.AdminSite):
         if not models_by_table:
             return response
 
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute("SELECT tbl, stat FROM sqlite_stat1")
-                for table, stat in cursor.fetchall():
-                    try:
-                        count = int(str(stat).split()[0])
-                    except (IndexError, TypeError, ValueError):
-                        continue
-                    self._set_model_object_count(
-                        models_by_table,
-                        table,
-                        count,
-                        title=f"Approximate count from SQLite stats: {count:,}",
-                    )
-                    models_by_table.pop(table, None)
-        except DatabaseError:
-            pass
+        from archivebox.misc.db import approximate_row_counts
+
+        for table, count in approximate_row_counts(connection).items():
+            if table not in models_by_table:
+                continue
+            self._set_model_object_count(
+                models_by_table,
+                table,
+                count,
+                title=f"Approximate count from database stats: {count:,}",
+            )
+            models_by_table.pop(table, None)
 
         for table in list(models_by_table):
             try:
@@ -142,9 +186,9 @@ def register_admin_site():
 
     # Register admin views for each app
     # (Previously handled by ABX plugin system, now called directly)
+    from archivebox.api.admin import register_admin as register_api_admin
     from archivebox.core.admin import register_admin as register_core_admin
     from archivebox.crawls.admin import register_admin as register_crawls_admin
-    from archivebox.api.admin import register_admin as register_api_admin
     from archivebox.machine.admin import register_admin as register_machine_admin
     from archivebox.personas.admin import register_admin as register_personas_admin
     from archivebox.workers.admin import register_admin as register_workers_admin

archivebox/core/admin_snapshots.py

@@ -30,7 +30,9 @@ from archivebox.plugins.discovery import get_plugin_icon, get_plugin_name, get_p
 from archivebox.base_models.admin import BaseModelAdmin, ConfigEditorMixin
 
 from archivebox.core.models import Tag, Snapshot, ArchiveResult
+from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
+from archivebox.core.preview_util import EXTENSION_SCREENSHOT_PLUGIN
 from archivebox.progressmonitor.views import progress_endpoint
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
@@ -41,7 +43,6 @@ from archivebox.core.permissions import (
 from archivebox.core.widgets import TagEditorWidget, InlineTagEditorWidget
 
 
-# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
 GLOBAL_CONTEXT = {}
 
 SNAPSHOT_PERMISSION_META = PERMISSIONS_META
@@ -70,13 +71,6 @@ class SnapshotActionForm(ActionForm):
 
         return [name.strip() for name in tags_str.split(",") if name.strip()]
 
-    # TODO: allow selecting actions for specific extractor plugins? is this useful?
-    # plugin = forms.ChoiceField(
-    #     choices=ArchiveResult.PLUGIN_CHOICES,
-    #     required=False,
-    #     widget=forms.MultileChoiceField(attrs={'class': "form-control"})
-    # )
-
 
 class TagNameListFilter(admin.SimpleListFilter):
     title = "By tag name"
@@ -107,9 +101,7 @@ class SnapshotPermissionsListFilter(admin.SimpleListFilter):
     def queryset(self, request, queryset):
         value = self.value()
         if value:
-            return queryset.filter(
-                Q(permissions=value) | (Q(permissions__isnull=True) & Q(crawl__permissions=value)),
-            )
+            return queryset.filter(permissions=value)
         return queryset
 
 
@@ -229,8 +221,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                         total_results=ArchiveResult.snapshot_count_expr(),
                         matching_results=ArchiveResult.snapshot_count_expr(status=status),
                     ).filter(matching_results__gt=F("total_results") / 2)
-                    queryset._archivebox_count_hint = "model_estimate"
-                    queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
                     return queryset
 
                 # Rare statuses are faster status-first: use the
@@ -262,13 +252,15 @@ class SnapshotChangeList(SearchResultsChangeList):
         rows = (
             ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
             .order_by("snapshot_id", "plugin")
-            .values_list("snapshot_id", "plugin", "status", "output_size")
+            .values_list("snapshot_id", "plugin", "status", "output_size", "output_files")
         )
-        for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+        for snapshot_id, plugin, status, output_size, output_files in rows.iterator(chunk_size=1000):
             if plugin in seen_plugins[snapshot_id]:
                 continue
             seen_plugins[snapshot_id].add(plugin)
-            results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+            results_by_snapshot[snapshot_id].append(
+                SimpleNamespace(plugin=plugin, status=status, output_size=output_size, output_files=output_files),
+            )
 
         for obj in self.result_list:
             obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
@@ -277,13 +269,7 @@ class SnapshotChangeList(SearchResultsChangeList):
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
             self.full_result_count = self.result_count
-        else:
-            self.full_result_count = self.model_admin.get_paginator(
-                request,
-                self.model._default_manager.all().order_by("-id"),
-                self.list_per_page,
-            ).count
-        self.show_full_result_count = True
+            self.show_full_result_count = True
         self._attach_archiveresult_summaries()
 
 
@@ -320,10 +306,7 @@ class SnapshotAdminForm(forms.ModelForm):
         instance = super().save(commit=False)
         permissions = self.cleaned_data["permissions_config"]
         config = dict(instance.config or {})
-        if permissions == instance.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
         instance.config = config
 
         # Handle tags_editor field
@@ -482,7 +465,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     paginator = AcceleratedPaginator
 
     save_on_top = True
-    show_full_result_count = False
+    show_full_result_count = True
 
     def get_changelist(self, request, **kwargs):
         return SnapshotChangeList
@@ -578,12 +561,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if permissions not in dict(PERMISSIONS_CHOICES):
             return HttpResponseBadRequest("Invalid permissions value")
 
-        snapshot = get_object_or_404(Snapshot.objects.select_related("crawl"), pk=object_id)
+        snapshot = get_object_or_404(Snapshot, pk=object_id)
         config = dict(snapshot.config or {})
-        if permissions == snapshot.crawl.permissions:
-            config.pop("PERMISSIONS", None)
-        else:
-            config["PERMISSIONS"] = permissions
+        config["PERMISSIONS"] = permissions
 
         # Keep the quick-edit write to one targeted UPDATE so SQLite only holds
         # the write lock for the permission/config change itself. safe_update()
@@ -606,18 +586,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         now = timezone.now()
         updated = 0
         batch = []
-        snapshots = (
-            queryset.select_related(None)
-            .select_related("crawl")
-            .only("id", "config", "crawl__id", "crawl__permissions")
-            .prefetch_related(None)
-        )
+        snapshots = queryset.select_related(None).only("id", "config").prefetch_related(None)
         for snapshot in snapshots.iterator(chunk_size=500):
             config = dict(snapshot.config or {})
-            if permissions == snapshot.crawl.permissions:
-                config.pop("PERMISSIONS", None)
-            else:
-                config["PERMISSIONS"] = permissions
+            config["PERMISSIONS"] = permissions
             snapshot.config = config
             snapshot.modified_at = now
             batch.append(snapshot)
@@ -638,12 +610,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             if retried:
                 messages.success(
                     request,
-                    f"Queued {retried} failed/skipped extractors for retry on this snapshot.",
+                    f"Queued {retried} failed extractors for retry on this snapshot.",
                 )
             else:
                 messages.info(
                     request,
-                    "No failed/skipped extractors were found on this snapshot.",
+                    "No failed extractors were found on this snapshot.",
                 )
 
         return redirect(snapshot.admin_change_url)
@@ -665,13 +637,26 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                         "plugin",
                         "status",
                         "output_size",
+                        "output_files",
+                    ),
+                ),
+            )
+        else:
+            prefetches.append(
+                Prefetch(
+                    "crawl",
+                    queryset=Crawl.objects.select_related("created_by").only(
+                        "id",
+                        "created_by_id",
+                        "created_by__id",
+                        "created_by__username",
                     ),
                 ),
             )
 
-        qs = super().get_queryset(request).select_related("crawl__created_by")
+        qs = super().get_queryset(request)
         if is_change_view:
-            qs = qs.defer("notes")
+            qs = qs.select_related("crawl__created_by").defer("notes")
         else:
             qs = qs.only(
                 "id",
@@ -685,13 +670,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
                 "fs_version",
                 "output_size",
                 "permissions",
-                "crawl__id",
-                "crawl__permissions",
-                "crawl__persona_id",
-                "crawl__status",
-                "crawl__created_by_id",
-                "crawl__created_by__id",
-                "crawl__created_by__username",
             )
         qs = qs.prefetch_related(*prefetches)
         if needs_files_sort:
@@ -707,10 +685,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def permissions_badge(self, obj):
         permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
-            if obj.permissions:
-                permissions = obj.permissions
-            else:
-                permissions = obj.crawl.permissions
+            permissions = obj.permissions
         permissions = normalize_permissions(permissions)
         icon, label, fg, bg = SNAPSHOT_PERMISSION_META[permissions]
         menu_items = format_html_join(
@@ -765,31 +740,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         html = Template("""{{bookmarked_date}} (<code>{{timestamp}}</code>)""")
         return mark_safe(html.render(context))
 
-        # pretty_time = obj.bookmarked.strftime('%Y-%m-%d %H:%M:%S')
-        # return f'{pretty_time} ({obj.timestamp})'
-
-    # TODO: figure out a different way to do this, you cant nest forms so this doenst work
-    # def action(self, obj):
-    #     # csrfmiddlewaretoken: Wa8UcQ4fD3FJibzxqHN3IYrrjLo4VguWynmbzzcPYoebfVUnDovon7GEMYFRgsh0
-    #     # action: update_snapshots
-    #     # select_across: 0
-    #     # _selected_action: 76d29b26-2a88-439e-877c-a7cca1b72bb3
-    #     return format_html(
-    #         '''
-    #             <form action="/admin/core/snapshot/" method="post" onsubmit="e => e.stopPropagation()">
-    #                 <input type="hidden" name="csrfmiddlewaretoken" value="{}">
-    #                 <input type="hidden" name="_selected_action" value="{}">
-    #                 <button name="update_snapshots">Check</button>
-    #                 <button name="update_titles">Pull title + favicon</button>
-    #                 <button name="update_snapshots">Update</button>
-    #                 <button name="overwrite_snapshots">Re-Archive (overwrite)</button>
-    #                 <button name="delete_snapshots">Permanently delete</button>
-    #             </form>
-    #         ''',
-    #         csrf.get_token(self.request),
-    #         obj.pk,
-    #     )
-
     @admin.display(description="")
     def admin_actions(self, obj):
         summary_url = self.get_snapshot_view_url(obj)
@@ -959,30 +909,54 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         config = request.archivebox_config
         results = self._get_prefetched_results(obj)
         if results is not None:
-            has_screenshot = any(r.plugin == "screenshot" for r in results)
-            has_favicon = any(r.plugin == "favicon" for r in results)
+            results = [r for r in results if r.plugin in ("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")]
         else:
-            available_plugins = set(obj.archiveresult_set.filter(plugin__in=("screenshot", "favicon")).values_list("plugin", flat=True))
-            has_screenshot = "screenshot" in available_plugins
-            has_favicon = "favicon" in available_plugins
+            results = list(
+                obj.archiveresult_set.filter(plugin__in=("screenshot", EXTENSION_SCREENSHOT_PLUGIN, "favicon")).only(
+                    "snapshot_id",
+                    "plugin",
+                    "status",
+                    "output_files",
+                    "output_str",
+                ),
+            )
 
-        if not has_screenshot and not has_favicon:
+        def result_output_path(result, filename: str) -> str | None:
+            output_files = result.output_files or {}
+            file_info = output_files.get(filename)
+            if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+                return None
+            if file_info.get("root_relative"):
+                return filename
+            return f"{result.plugin}/{filename}"
+
+        def result_urls(plugin: str, filenames: tuple[str, ...]) -> list[str]:
+            urls: list[str] = []
+            for result in results:
+                if result.plugin != plugin or result.status != ArchiveResult.StatusChoices.SUCCEEDED:
+                    continue
+                for filename in filenames:
+                    output_path = result_output_path(result, filename)
+                    if output_path:
+                        urls.append(build_snapshot_url(str(obj.id), output_path, request=request, config=config))
+            return urls
+
+        screenshot_urls = result_urls("screenshot", ("screenshot.png",))
+        extension_screenshot_urls = result_urls(EXTENSION_SCREENSHOT_PLUGIN, ("screenshot-1.png", "screenshot.png"))
+        favicon_urls = result_urls("favicon", ("favicon.ico",))
+
+        if not screenshot_urls and not extension_screenshot_urls and not favicon_urls:
             return None
 
-        if has_screenshot:
-            img_url = build_snapshot_url(str(obj.id), "screenshot/screenshot.png", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "screenshot.png", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config),
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+        all_screenshot_urls = [*screenshot_urls, *extension_screenshot_urls]
+        if all_screenshot_urls:
+            img_url = all_screenshot_urls[0]
+            fallbacks = [*all_screenshot_urls[1:], *favicon_urls]
             img_alt = "Screenshot"
             preview_class = "screenshot"
         else:
-            img_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-            fallbacks = [
-                build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config),
-            ]
+            img_url = favicon_urls[0]
+            fallbacks = favicon_urls[1:]
             img_alt = "Favicon"
             preview_class = "favicon"
 
@@ -1000,6 +974,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             "preview_class": preview_class,
             "onerror_js": onerror_js,
             "fallback_list": fallback_list,
+            "favicon_url": favicon_urls[0] if favicon_urls else "",
+            "favicon_fallback_list": ",".join(favicon_urls[1:]),
         }
 
     @admin.display(description="", empty_value="")
@@ -1008,10 +984,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if not preview:
             return ""
 
-        request = self.request
-        config = request.archivebox_config
-        favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
-        fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
+        favicon_url = preview.get("favicon_url") or ""
+        if not favicon_url:
+            return ""
+        fallback_list = preview.get("favicon_fallback_list") or ""
         onerror_js = (
             "this.dataset.fallbacks && this.dataset.fallbacks.length ? "
             "(this.src=this.dataset.fallbacks.split(',').shift(), "
@@ -1140,9 +1116,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             mark_safe("".join(output)),
         )
 
-    @admin.display(
-        # ordering='archiveresult_count'
-    )
+    @admin.display()
     def size(self, obj):
         request = self.request
         config = request.archivebox_config
@@ -1392,11 +1366,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         extra_context["snapshot_is_grid_view"] = True
         return self.changelist_view(request, extra_context=extra_context)
 
-    # for debugging, uncomment this to print all requests:
-    # def changelist_view(self, request, extra_context=None):
-    #     print('[*] Got request', request.method, request.POST)
-    #     return super().changelist_view(request, extra_context=None)
-
     @admin.action(
         description="🔁 Redo Failed",
     )
@@ -1408,10 +1377,10 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if queued:
             messages.success(
                 request,
-                f"Queued {queued} failed/skipped extractors for retry. The background runner will process them.",
+                f"Queued {queued} failed extractors for retry. The background runner will process them.",
             )
         else:
-            messages.info(request, "No failed/skipped extractors were found in the selected snapshots.")
+            messages.info(request, "No failed extractors were found in the selected snapshots.")
 
     @admin.action(
         description="🆕 Archive Now",

archivebox/core/apps.py

@@ -20,6 +20,14 @@ class CoreConfig(AppConfig):
 
         ModelWithOutputDir.register_delete_signal()
 
+        # SQLite ignores VARCHAR(n) limits but PostgreSQL enforces them; clamp
+        # CharField values on save so writes behave the same on both backends.
+        from django.db.models.signals import pre_save
+
+        from archivebox.misc.db import truncate_overlong_charfields
+
+        pre_save.connect(truncate_overlong_charfields, dispatch_uid="archivebox_truncate_overlong_charfields")
+
         # Import models to register state machines with the registry
         # Skip during makemigrations to avoid premature state machine access
         if "makemigrations" not in sys.argv:

archivebox/core/asgi.py

@@ -7,10 +7,11 @@ For more information on this file, see
 https://docs.djangoproject.com/en/stable/howto/deployment/asgi/
 """
 
-from archivebox.config.django import setup_django
 from django.core.asgi import get_asgi_application
 
-setup_django(in_memory_db=False, check_db=True)
+from archivebox.config.django import setup_django
+
+setup_django(check_db=True)
 
 
 def _patch_thread_sensitive_context_shutdown() -> None:
@@ -73,16 +74,3 @@ _patch_thread_sensitive_context_shutdown()
 
 # Standard Django ASGI application (no websockets/channels needed)
 application = get_asgi_application()
-
-# If websocket support is needed later, install channels and use:
-# from channels.routing import ProtocolTypeRouter, URLRouter
-# from channels.auth import AuthMiddlewareStack
-# from channels.security.websocket import AllowedHostsOriginValidator
-# from archivebox.core.routing import websocket_urlpatterns
-#
-# application = ProtocolTypeRouter({
-#     "http": get_asgi_application(),
-#     "websocket": AllowedHostsOriginValidator(
-#         AuthMiddlewareStack(URLRouter(websocket_urlpatterns))
-#     ),
-# })

archivebox/core/context_processors.py

@@ -1,9 +1,21 @@
+from pathlib import Path
+
 from archivebox.config import VERSION
 from archivebox.config.version import get_COMMIT_HASH
 
 
+def get_static_cache_key() -> str:
+    """Version the admin stylesheet even when the checkout has uncommitted edits."""
+    base_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
+    admin_css_path = Path(__file__).resolve().parent.parent / "templates" / "static" / "admin.css"
+    try:
+        return f"{base_key}-{admin_css_path.stat().st_mtime_ns}"
+    except OSError:
+        return base_key
+
+
 def archivebox_globals(request):
     return {
         "VERSION": VERSION,
-        "STATIC_CACHE_KEY": (get_COMMIT_HASH() or VERSION or "dev").strip(),
+        "STATIC_CACHE_KEY": get_static_cache_key(),
     }

archivebox/core/forms.py

@@ -1,17 +1,17 @@
 __package__ = "archivebox.core"
 
 import re
-from decimal import Decimal, InvalidOperation, ROUND_CEILING
+from decimal import ROUND_CEILING, Decimal, InvalidOperation
 
 from django import forms
 
-from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
-from taggit.utils import edit_string_for_tags, parse_tags
 from archivebox.base_models.admin import KeyValueWidget
-from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.config.common import get_config, parse_delete_after
-from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions, is_admin_user
+from archivebox.core.permissions import PERMISSIONS_CHOICES, PERMISSIONS_PUBLIC, filter_personas_by_permissions
 from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
+from archivebox.crawls.schedule_util import validate_schedule
+from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
+from archivebox.personas.models import Persona
 from archivebox.plugins.discovery import get_plugins
 from archivebox.plugins.forms import (
     PLUGIN_GROUP_DEFINITIONS,
@@ -19,7 +19,6 @@ from archivebox.plugins.forms import (
     PluginConfigFormMixin,
     get_choice_field,
 )
-from archivebox.personas.models import Persona
 
 DEPTH_CHOICES = (
     ("0", "depth = 0 (archive just these URLs)"),
@@ -30,22 +29,83 @@ DEPTH_CHOICES = (
 )
 
 
+def _split_strip(value: str, delimiter: str) -> list[str]:
+    return [part.strip() for part in value.split(delimiter) if part.strip()]
+
+
+def parse_tag_string(value: str | None) -> list[str]:
+    """Parse the legacy tag-editing format used by the admin tag field."""
+    if not value:
+        return []
+
+    if "," not in value and '"' not in value:
+        return sorted(set(_split_strip(value, " ")))
+
+    tags: list[str] = []
+    buffer: list[str] = []
+    deferred_chunks: list[str] = []
+    saw_unquoted_comma = False
+    in_quote = False
+    chars = iter(value)
+
+    try:
+        while True:
+            char = next(chars)
+            if char == '"':
+                if buffer:
+                    deferred_chunks.append("".join(buffer))
+                    buffer = []
+                in_quote = True
+                char = next(chars)
+                while char != '"':
+                    buffer.append(char)
+                    char = next(chars)
+                tag = "".join(buffer).strip()
+                if tag:
+                    tags.append(tag)
+                buffer = []
+                in_quote = False
+            else:
+                if not saw_unquoted_comma and char == ",":
+                    saw_unquoted_comma = True
+                buffer.append(char)
+    except StopIteration:
+        if buffer:
+            if in_quote and "," in buffer:
+                saw_unquoted_comma = True
+            deferred_chunks.append("".join(buffer))
+
+    delimiter = "," if saw_unquoted_comma else " "
+    for chunk in deferred_chunks:
+        tags.extend(_split_strip(chunk, delimiter))
+
+    return sorted(set(tags))
+
+
+def edit_string_for_tag_names(tags) -> str:
+    names = []
+    for tag in tags:
+        name = tag.name
+        names.append(f'"{name}"' if "," in name or " " in name else name)
+    return ", ".join(sorted(names))
+
+
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
     allow_crawl_execution_config_fields = False
 
     # Basic fields
     url = forms.CharField(
         label="URLs",
-        strip=True,
+        strip=False,
         widget=forms.Textarea(
             attrs={
                 "data-url-regex": URL_REGEX.pattern,
                 "placeholder": (
-                    "Enter URLs to archive, as one per line, CSV, JSON, or embedded in text "
-                    "(e.g. markdown, HTML, etc.). Examples:\n"
-                    "https://example.com\n"
-                    "https://news.ycombinator.com,https://news.google.com\n"
-                    "[ArchiveBox](https://github.com/ArchiveBox/ArchiveBox)"
+                    "Enter URL(s) to archive. Any format is ok: one per line, CSV, JSON, RSS, embedded in text, etc.\n\n"
+                    "Examples:\n\n"
+                    "https://example.com\n\n"
+                    "https://news.ycombinator.com,https://news.google.com\n\n"
+                    "Or any text-based format [containing URLs](https://github.com/ArchiveBox/ArchiveBox)..."
                 ),
             },
         ),
@@ -238,7 +298,10 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def __init__(self, *args, **kwargs):
         self.request = kwargs.pop("request", None)
-        self.can_override_crawl_config = bool(self.request and is_admin_user(self.request))
+        self.can_override_crawl_config = False
+        if self.request is not None:
+            user = self.request.user
+            self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
         default_persona = Persona.get_or_create_default()
@@ -314,12 +377,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 
     def clean_url(self):
         value = self.cleaned_data.get("url") or ""
-        valid_urls = []
-        for url in find_all_urls(value):
-            valid_urls.append(url)
-        if not valid_urls:
+        if not list(find_all_urls(value)):
             raise forms.ValidationError("Enter at least one valid URL.")
-        return "\n".join(valid_urls)
+        return value
 
     def clean_url_filters(self):
         from archivebox.crawls.models import Crawl
@@ -425,7 +485,7 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
 class TagWidget(forms.TextInput):
     def format_value(self, value):
         if value is not None and not isinstance(value, str):
-            value = edit_string_for_tags(value)
+            value = edit_string_for_tag_names(value)
         return super().format_value(value)
 
 
@@ -434,12 +494,7 @@ class TagField(forms.CharField):
 
     def clean(self, value):
         value = super().clean(value)
-        try:
-            return parse_tags(value)
-        except ValueError:
-            raise forms.ValidationError(
-                "Please provide a comma-separated list of tags.",
-            )
+        return parse_tag_string(value)
 
     def has_changed(self, initial, data):
         # Always return False if the field is disabled since self.bound_data

archivebox/core/middleware.py

@@ -3,35 +3,35 @@ __package__ = "archivebox.core"
 import ipaddress
 import re
 from pathlib import Path
-from django.utils import timezone
+
+from django.conf import settings
 from django.contrib.auth.middleware import RemoteUserMiddleware
 from django.contrib.auth.models import AnonymousUser
-from django.core.exceptions import ImproperlyConfigured
-from django.shortcuts import redirect
 from django.contrib.staticfiles import finders
-from django.utils.http import http_date
+from django.core.exceptions import ImproperlyConfigured
 from django.http import HttpResponseForbidden, HttpResponseNotModified
+from django.shortcuts import redirect
+from django.utils import timezone
+from django.utils.http import http_date
 
-from archivebox.config.common import get_config
 from archivebox.config import VERSION
+from archivebox.config.common import get_config
 from archivebox.config.version import get_COMMIT_HASH
 from archivebox.core.routes_util import (
-    build_snapshot_url,
     build_admin_url,
+    build_snapshot_url,
     build_web_url,
-    get_api_host,
     get_admin_host,
+    get_api_host,
     get_base_host,
     get_listen_host,
     get_listen_subdomain,
-    get_public_host,
     get_web_host,
     host_matches,
     is_snapshot_subdomain,
     split_host_port,
 )
-from archivebox.core.views import SnapshotHostView, OriginalDomainHostView
-
+from archivebox.core.views import OriginalDomainHostView, SnapshotHostView
 
 ADMIN_LOGIN_HINT_COOKIE = "archivebox_admin_logged_in"
 
@@ -42,7 +42,7 @@ def _admin_login_hint_cookie_domain(config) -> str | None:
     NOTE: this cookie carries only the single bit "user is logged in on
     admin somewhere"; it MUST NOT be confused with the session cookie,
     which stays admin-host-scoped (see core/settings.py
-    SESSION_COOKIE_DOMAIN comment — admin/public is a security boundary).
+    SESSION_COOKIE_DOMAIN comment — admin/web is a security boundary).
 
     Returns the hostname portion of ``get_base_host`` (which respects
     ``BASE_URL`` and falls back to the local-bind mapping). Strips the
@@ -73,7 +73,6 @@ def detect_timezone(request, activate: bool = True):
         tz = timezone.get_fixed_timezone(int(gmt_offset))
         if activate:
             timezone.activate(tz)
-    # print('GMT_OFFSET', gmt_offset, tz)
     return tz
 
 
@@ -85,6 +84,39 @@ def TimezoneMiddleware(get_response):
     return middleware
 
 
+def AdminCookieIsolationMiddleware(get_response):
+    def middleware(request):
+        response = get_response(request)
+
+        config = request.__dict__.get("archivebox_config")
+        if config is None or config.SERVER_SECURITY_MODE == "auto":
+            from archivebox.config.common import get_request_config
+
+            config = get_request_config(request, resolve_plugins=False)
+        if not config.USES_SUBDOMAIN_ROUTING:
+            return response
+
+        if not config.BASE_URL and request.path.startswith("/admin/"):
+            return response
+
+        request_host = (request.get_host() or "").lower()
+        request_hostname, _request_port = split_host_port(request_host)
+        if host_matches(request_hostname, get_admin_host(config=config)):
+            return response
+
+        if host_matches(request_hostname, get_web_host(config=config)):
+            for cookie_name in tuple(response.cookies.keys()):
+                if cookie_name != ADMIN_LOGIN_HINT_COOKIE:
+                    response.cookies.pop(cookie_name, None)
+            return response
+
+        response.cookies.pop(settings.SESSION_COOKIE_NAME, None)
+        response.cookies.pop(settings.CSRF_COOKIE_NAME, None)
+        return response
+
+    return middleware
+
+
 def CacheControlMiddleware(get_response):
     snapshot_path_re = re.compile(r"^/[^/]+/\\d{8}/[^/]+/[0-9a-fA-F-]{8,36}/")
     static_cache_key = (get_COMMIT_HASH() or VERSION or "dev").strip()
@@ -117,15 +149,15 @@ def CacheControlMiddleware(get_response):
                     response.headers["Last-Modified"] = http_date(mtime)
                 return response
 
-        if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
-            if not response.get("Cache-Control"):
-                config = request.__dict__.get("archivebox_config")
-                if config is None:
-                    config = get_config(resolve_plugins=False)
-                    request.archivebox_config = config
-                policy = "private" if config.PERMISSIONS == "private" else "public"
-                response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
-                # print('Set Cache-Control header to', response['Cache-Control'])
+        if ("/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path)) and not response.get(
+            "Cache-Control",
+        ):
+            config = request.__dict__.get("archivebox_config")
+            if config is None:
+                config = get_config(resolve_plugins=False)
+                request.archivebox_config = config
+            policy = "private" if config.PERMISSIONS == "private" else "public"
+            response["Cache-Control"] = f"{policy}, max-age=60, stale-while-revalidate=300"
         return response
 
     return middleware
@@ -137,9 +169,28 @@ def ServerSecurityModeMiddleware(get_response):
 
     def middleware(request):
         config = request.__dict__.get("archivebox_config")
-        if config is None:
-            config = get_config(resolve_plugins=False)
-            request.archivebox_config = config
+        if config is None or config.SERVER_SECURITY_MODE == "auto":
+            base_config = config or get_config(resolve_plugins=False)
+            if base_config.SERVER_SECURITY_MODE == "auto" and request.method.upper() not in allowed_methods:
+                request_host, _request_port = split_host_port((request.get_host() or "").lower())
+                base_host, _base_port = split_host_port(get_base_host(config=base_config))
+                admin_host, _admin_port = split_host_port(get_admin_host(config=base_config))
+                api_host, _api_port = split_host_port(get_api_host(config=base_config))
+                web_host, _web_port = split_host_port(get_web_host(config=base_config))
+                control_hosts = {host for host in (base_host, admin_host, api_host, web_host) if host}
+                first_run_setup_request = not base_config.BASE_URL and (
+                    request.path == "/admin/login/"
+                    or (
+                        bool(getattr(request.user, "is_superuser", False))
+                        and re.fullmatch(r"/admin/machine/machine/[0-9a-f-]{32,36}/change/", request.path)
+                    )
+                )
+                if control_hosts and request_host not in control_hosts and not first_run_setup_request:
+                    return HttpResponseForbidden("ArchiveBox is running with the control plane disabled on this host.")
+
+            from archivebox.config.common import get_request_config
+
+            config = get_request_config(request, resolve_plugins=False)
 
         if config.CONTROL_PLANE_ENABLED:
             return get_response(request)
@@ -163,6 +214,10 @@ def HostRoutingMiddleware(get_response):
     snapshot_path_re = re.compile(
         r"^/(?P<username>[^/]+)/(?P<date>\d{4}(?:\d{2})?(?:\d{2})?)/(?P<domain>[^/]+)/(?P<snapshot_id>[0-9a-fA-F-]{8,36})(?:/(?P<path>.*))?$",
     )
+    snapshot_replay_path_re = re.compile(
+        r"^/snapshot/(?P<snapshot_id>[0-9a-fA-F-]{8,36})(?:/(?P<path>.*))?$",
+    )
+    original_replay_path_re = re.compile(r"^/original/(?P<domain>[^/]+)(?:/(?P<path>.*))?$")
 
     def middleware(request):
         if request.path in {"/health", "/health/"}:
@@ -176,7 +231,6 @@ def HostRoutingMiddleware(get_response):
         admin_host = get_admin_host(config=config)
         web_host = get_web_host(config=config)
         api_host = get_api_host(config=config)
-        public_host = get_public_host(config=config)
         listen_host = get_listen_host(config=config)
         subdomain = get_listen_subdomain(request_host, config=config)
 
@@ -213,18 +267,38 @@ def HostRoutingMiddleware(get_response):
         if config.USES_SUBDOMAIN_ROUTING and not config.BASE_URL:
             return get_response(request)
 
+        if config.USES_SUBDOMAIN_ROUTING:
+            snapshot_replay_match = snapshot_replay_path_re.match(request.path)
+            if snapshot_replay_match:
+                target = build_snapshot_url(
+                    snapshot_replay_match.group("snapshot_id"),
+                    (snapshot_replay_match.group("path") or "").strip("/"),
+                    request=request,
+                )
+                if request.META.get("QUERY_STRING"):
+                    target = f"{target}?{request.META['QUERY_STRING']}"
+                return redirect(target)
+
+            original_replay_match = original_replay_path_re.match(request.path)
+            if original_replay_match:
+                view = OriginalDomainHostView.as_view()
+                return view(
+                    request,
+                    domain=original_replay_match.group("domain"),
+                    path=(original_replay_match.group("path") or "").strip("/"),
+                )
+
         if not config.USES_SUBDOMAIN_ROUTING:
             if host_matches(request_host, listen_host):
                 return get_response(request)
 
             req_host, req_port = split_host_port(request_host)
             listen_host_only, listen_port = split_host_port(listen_host)
-            if req_host.endswith(f".{listen_host_only}"):
-                if not listen_port or not req_port or listen_port == req_port:
-                    target = build_web_url(request.path, request=request)
-                    if request.META.get("QUERY_STRING"):
-                        target = f"{target}?{request.META['QUERY_STRING']}"
-                    return redirect(target)
+            if req_host.endswith(f".{listen_host_only}") and (not listen_port or not req_port or listen_port == req_port):
+                target = build_web_url(request.path, request=request)
+                if request.META.get("QUERY_STRING"):
+                    target = f"{target}?{request.META['QUERY_STRING']}"
+                return redirect(target)
 
             return get_response(request)
 
@@ -278,12 +352,6 @@ def HostRoutingMiddleware(get_response):
             request._cached_user = request.user
             return get_response(request)
 
-        if host_matches(request_host, public_host):
-            if request.COOKIES.get(ADMIN_LOGIN_HINT_COOKIE) == "1" and (request.path == "/public" or request.path.startswith("/public/")):
-                target = build_admin_url("/admin/core/snapshot/", request=request)
-                return redirect(target)
-            return get_response(request)
-
         if subdomain:
             view = OriginalDomainHostView.as_view()
             return view(request, domain=subdomain, path=request.path.lstrip("/"))

archivebox/core/migrations/0004_auto_20200713_1552.py

@@ -9,6 +9,12 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        # The original schema allowed NULL timestamps. Use the already-unique
+        # 32-character snapshot ID before enforcing the later NOT NULL field.
+        migrations.RunSQL(
+            sql="UPDATE core_snapshot SET timestamp = id WHERE timestamp IS NULL",
+            reverse_sql=migrations.RunSQL.noop,
+        ),
         migrations.AlterField(
             model_name="snapshot",
             name="timestamp",

archivebox/core/migrations/0007_archiveresult.py

@@ -1,9 +1,11 @@
 # Generated by Django 3.0.8 on 2020-11-04 12:25
 
 import json
+from datetime import timezone
 from pathlib import Path
 
 from django.db import migrations, models
+from django.utils.dateparse import parse_datetime
 import django.db.models.deletion
 
 # Handle old vs new import paths
@@ -30,6 +32,13 @@ except AttributeError:
     JSONField = jsonfield.JSONField
 
 
+def parse_legacy_datetime(value):
+    parsed = parse_datetime(value) if isinstance(value, str) else value
+    if parsed is not None and parsed.tzinfo is None:
+        return parsed.replace(tzinfo=timezone.utc)
+    return parsed
+
+
 def forwards_func(apps, schema_editor):
     Snapshot = apps.get_model("core", "Snapshot")
     ArchiveResult = apps.get_model("core", "ArchiveResult")
@@ -55,8 +64,8 @@ def forwards_func(apps, schema_editor):
                         pwd=result["pwd"],
                         cmd=result.get("cmd") or [],
                         cmd_version=result.get("cmd_version") or "unknown",
-                        start_ts=result["start_ts"],
-                        end_ts=result["end_ts"],
+                        start_ts=parse_legacy_datetime(result["start_ts"]),
+                        end_ts=parse_legacy_datetime(result["end_ts"]),
                         status=result["status"],
                         output=result.get("output") or "null",
                     )

archivebox/core/migrations/0023_upgrade_to_0_9_0.py

@@ -19,6 +19,19 @@ def get_table_columns(table_name):
     return {row[1] for row in cursor.fetchall()}
 
 
+def assert_rebuild_row_count(cursor, source_table, destination_table, expected_count=None):
+    """Refuse to drop a legacy table unless every source row was copied."""
+    if expected_count is None:
+        cursor.execute(f"SELECT COUNT(*) FROM {source_table}")
+        expected_count = cursor.fetchone()[0]
+    cursor.execute(f"SELECT COUNT(*) FROM {destination_table}")
+    destination_count = cursor.fetchone()[0]
+    if destination_count != expected_count:
+        raise RuntimeError(
+            f"Refusing to replace {source_table}: copied {destination_count} of {expected_count} rows",
+        )
+
+
 def normalize_cmd(cmd):
     if not cmd:
         return "[]"
@@ -47,6 +60,14 @@ def normalize_status(status):
 
 def upgrade_core_tables(apps, schema_editor):
     """Upgrade core tables from v0.7.2 or v0.8.6rc0 to v0.9.0."""
+    # sqlite-only legacy repair/rebuild. On postgres this raw SQL (PRAGMA,
+    # sqlite_master, INSERT OR IGNORE, DATETIME, table-rebuild dance) is both
+    # invalid and unnecessary: a postgres database can never contain legacy
+    # data at this point. The final _pg_sync_schema op resyncs the real
+    # postgres schema to migration state instead.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
     from archivebox.uuid_compat import uuid7
 
     cursor = connection.cursor()
@@ -91,6 +112,7 @@ def upgrade_core_tables(apps, schema_editor):
             output VARCHAR(1024),
             created_at DATETIME,
             modified_at DATETIME,
+            notes TEXT NOT NULL DEFAULT '',
 
             FOREIGN KEY (snapshot_id) REFERENCES core_snapshot(id) ON DELETE CASCADE
         );
@@ -115,6 +137,8 @@ def upgrade_core_tables(apps, schema_editor):
             archiveresult_select_cols.append("created_at")
         if has_archiveresult_modified_at:
             archiveresult_select_cols.append("modified_at")
+        if "notes" in archiveresult_cols:
+            archiveresult_select_cols.append("notes")
 
         if has_uuid and not has_abid:
             # Migrating from v0.7.2+ (has uuid column)
@@ -124,6 +148,8 @@ def upgrade_core_tables(apps, schema_editor):
                 select_cols.append("created_at")
             if has_archiveresult_modified_at:
                 select_cols.append("modified_at")
+            if "notes" in archiveresult_cols:
+                select_cols.append("notes")
             cursor.execute(f"SELECT {', '.join(select_cols)} FROM core_archiveresult")
             old_records = cursor.fetchall()
             for i, record in enumerate(old_records, start=1):
@@ -139,8 +165,8 @@ def upgrade_core_tables(apps, schema_editor):
                     INSERT OR IGNORE INTO core_archiveresult_new (
                         id, uuid, snapshot_id, cmd, pwd, cmd_version,
                         start_ts, end_ts, status, extractor, output,
-                        created_at, modified_at
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        created_at, modified_at, notes
+                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 """,
                     (
                         values["id"],
@@ -156,6 +182,7 @@ def upgrade_core_tables(apps, schema_editor):
                         values["output"] or "",
                         values.get("created_at") or start_ts,
                         values.get("modified_at") or end_ts,
+                        values.get("notes") or "",
                     ),
                 )
                 if i % PROGRESS_EVERY == 0:
@@ -178,8 +205,8 @@ def upgrade_core_tables(apps, schema_editor):
                     INSERT OR IGNORE INTO core_archiveresult_new (
                         uuid, snapshot_id, cmd, pwd, cmd_version,
                         start_ts, end_ts, status, extractor, output,
-                        created_at, modified_at
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        created_at, modified_at, notes
+                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 """,
                     (
                         new_uuid,
@@ -194,6 +221,7 @@ def upgrade_core_tables(apps, schema_editor):
                         values["output"] or "",
                         values.get("created_at") or start_ts,
                         values.get("modified_at") or end_ts,
+                        values.get("notes") or "",
                     ),
                 )
                 if i % PROGRESS_EVERY == 0:
@@ -213,8 +241,8 @@ def upgrade_core_tables(apps, schema_editor):
                     INSERT OR IGNORE INTO core_archiveresult_new (
                         id, uuid, snapshot_id, cmd, pwd, cmd_version,
                         start_ts, end_ts, status, extractor, output,
-                        created_at, modified_at
-                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                        created_at, modified_at, notes
+                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                 """,
                     (
                         values["id"],
@@ -230,12 +258,14 @@ def upgrade_core_tables(apps, schema_editor):
                         values["output"] or "",
                         values.get("created_at") or start_ts,
                         values.get("modified_at") or end_ts,
+                        values.get("notes") or "",
                     ),
                 )
                 if i % PROGRESS_EVERY == 0:
                     print(f"        copied {i}/{len(old_records)} ArchiveResults...")
         print(f"      copied {len(old_records)} ArchiveResults")
 
+    assert_rebuild_row_count(cursor, "core_archiveresult", "core_archiveresult_new", row_count)
     cursor.execute("DROP TABLE IF EXISTS core_archiveresult;")
     cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult;")
 
@@ -288,13 +318,29 @@ def upgrade_core_tables(apps, schema_editor):
             snapshot_cols = get_table_columns("core_snapshot")
             has_added = "added" in snapshot_cols
             has_bookmarked_at = "bookmarked_at" in snapshot_cols
+            inferred_status_sql = """
+                CASE
+                    WHEN EXISTS (
+                        SELECT 1 FROM core_archiveresult
+                        WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                        AND core_archiveresult.status IN ('queued', 'started', 'backoff')
+                    )
+                    THEN 'queued'
+                    WHEN EXISTS (
+                        SELECT 1 FROM core_archiveresult
+                        WHERE core_archiveresult.snapshot_id = core_snapshot.id
+                    )
+                    THEN 'sealed'
+                    ELSE 'queued'
+                END
+            """
 
             if has_added and not has_bookmarked_at:
                 # Migrating from v0.7.2 (has added/updated fields)
                 print("      copying Snapshots from 0.7.x schema...")
                 # timestamp is the legacy bookmark/import timestamp and archive/{timestamp} identity.
                 # added is the DB row creation/import time, and updated was renamed to downloaded_at in 0.8.x.
-                cursor.execute("""
+                cursor.execute(f"""
                     INSERT OR IGNORE INTO core_snapshot_new (
                         id, url, timestamp, title,
                         bookmarked_at, created_at, modified_at, downloaded_at,
@@ -313,20 +359,7 @@ def upgrade_core_tables(apps, schema_editor):
                         COALESCE(added, CURRENT_TIMESTAMP) as created_at,
                         COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at,
                         updated as downloaded_at,
-                        CASE
-                            WHEN EXISTS (
-                                SELECT 1 FROM core_archiveresult
-                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
-                                AND core_archiveresult.status IN ('queued', 'started', 'backoff')
-                            )
-                            THEN 'queued'
-                            WHEN EXISTS (
-                                SELECT 1 FROM core_archiveresult
-                                WHERE core_archiveresult.snapshot_id = core_snapshot.id
-                            )
-                            THEN 'sealed'
-                            ELSE 'queued'
-                        END as status
+                        {inferred_status_sql} as status
                     FROM core_snapshot;
                 """)
                 print(f"      copied {cursor.rowcount} Snapshots")
@@ -344,6 +377,9 @@ def upgrade_core_tables(apps, schema_editor):
                 if has_crawl_id:
                     insert_cols.append("crawl_id")
                     select_cols.append("REPLACE(crawl_id, '-', '')")
+                if "parent_snapshot_id" in snapshot_cols:
+                    insert_cols.append("parent_snapshot_id")
+                    select_cols.append("REPLACE(parent_snapshot_id, '-', '')")
                 if has_status:
                     insert_cols.append("status")
                     select_cols.append(
@@ -365,9 +401,24 @@ def upgrade_core_tables(apps, schema_editor):
                         END
                         """,
                     )
+                else:
+                    insert_cols.append("status")
+                    select_cols.append(inferred_status_sql)
                 if has_retry_at:
                     insert_cols.append("retry_at")
                     select_cols.append("retry_at")
+                for field_name in (
+                    "depth",
+                    "fs_version",
+                    "config",
+                    "notes",
+                    "num_uses_succeeded",
+                    "num_uses_failed",
+                    "current_step",
+                ):
+                    if field_name in snapshot_cols:
+                        insert_cols.append(field_name)
+                        select_cols.append(field_name)
 
                 cursor.execute(f"""
                     INSERT OR IGNORE INTO core_snapshot_new ({", ".join(insert_cols)})
@@ -378,6 +429,7 @@ def upgrade_core_tables(apps, schema_editor):
             else:
                 print(f"Warning: Unexpected Snapshot schema - has_added={has_added}, has_bookmarked_at={has_bookmarked_at}")
 
+    assert_rebuild_row_count(cursor, "core_snapshot", "core_snapshot_new")
     cursor.execute("DROP TABLE IF EXISTS core_snapshot;")
     cursor.execute("ALTER TABLE core_snapshot_new RENAME TO core_snapshot;")
 
@@ -416,6 +468,7 @@ def upgrade_core_tables(apps, schema_editor):
         tag_has_data = cursor.fetchone()[0] > 0
 
         if tag_has_data:
+            tag_cols = get_table_columns("core_tag")
             cursor.execute("PRAGMA table_info(core_tag)")
             tag_id_type = None
             for row in cursor.fetchall():
@@ -427,22 +480,23 @@ def upgrade_core_tables(apps, schema_editor):
                 # v0.8.6rc0: Tag IDs are UUIDs, need to convert to INTEGER
                 print("      converting Tag IDs from UUID to integers...")
 
-                # Get all tags with their UUIDs
-                cursor.execute("SELECT id, name, slug, created_at, modified_at, created_by_id FROM core_tag ORDER BY name")
+                # Get all tags with their UUIDs and any available audit fields.
+                tag_select_cols = ["id", "name", "slug"]
+                tag_select_cols.extend(field for field in ("created_at", "modified_at", "created_by_id") if field in tag_cols)
+                cursor.execute(f"SELECT {', '.join(tag_select_cols)} FROM core_tag ORDER BY name")
                 tags = cursor.fetchall()
 
                 # Create mapping from old UUID to new INTEGER ID
                 uuid_to_int_map = {}
                 for i, tag in enumerate(tags, start=1):
-                    old_id, name, slug, created_at, modified_at, created_by_id = tag
+                    values = dict(zip(tag_select_cols, tag))
+                    old_id = values.pop("id")
                     uuid_to_int_map[old_id] = i
                     # Insert with new INTEGER ID
+                    insert_values = {"id": i, **values}
                     cursor.execute(
-                        """
-                        INSERT OR IGNORE INTO core_tag_new (id, name, slug, created_at, modified_at, created_by_id)
-                        VALUES (?, ?, ?, ?, ?, ?)
-                    """,
-                        (i, name, slug, created_at, modified_at, created_by_id),
+                        f"INSERT OR IGNORE INTO core_tag_new ({', '.join(insert_values)}) VALUES ({', '.join('%s' for _ in insert_values)})",
+                        tuple(insert_values.values()),
                     )
                     if i % PROGRESS_EVERY == 0:
                         print(f"        copied {i}/{len(tags)} Tags...")
@@ -463,23 +517,33 @@ def upgrade_core_tables(apps, schema_editor):
                             cursor.execute(
                                 """
                                 INSERT OR IGNORE INTO core_snapshot_tags (id, snapshot_id, tag_id)
-                                VALUES (?, ?, ?)
+                                VALUES (%s, %s, %s)
                             """,
                                 (st_id, snapshot_id, new_tag_id),
                             )
                         if i % PROGRESS_EVERY == 0:
                             print(f"        copied {i}/{len(snapshot_tags)} SnapshotTag rows...")
+                    cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
+                    copied_snapshot_tags = cursor.fetchone()[0]
+                    if copied_snapshot_tags != len(snapshot_tags):
+                        raise RuntimeError(
+                            f"Refusing to replace Tag IDs: copied {copied_snapshot_tags} of {len(snapshot_tags)} SnapshotTag rows",
+                        )
                 print(f"      copied {len(tags)} Tags")
             else:
-                # v0.7.2: Tag IDs are already INTEGER
-                print("      copying Tags from 0.7.x schema...")
-                cursor.execute("""
-                    INSERT OR IGNORE INTO core_tag_new (id, name, slug)
-                    SELECT id, name, slug
+                # v0.7.x and early v0.8.x use INTEGER IDs. Preserve audit
+                # metadata when the source schema has it.
+                print("      copying Tags with integer IDs...")
+                copy_cols = ["id", "name", "slug"]
+                copy_cols.extend(field for field in ("created_at", "modified_at", "created_by_id") if field in tag_cols)
+                cursor.execute(f"""
+                    INSERT OR IGNORE INTO core_tag_new ({", ".join(copy_cols)})
+                    SELECT {", ".join(copy_cols)}
                     FROM core_tag;
                 """)
                 print(f"      copied {cursor.rowcount} Tags")
 
+    assert_rebuild_row_count(cursor, "core_tag", "core_tag_new")
     cursor.execute("DROP TABLE IF EXISTS core_tag;")
     cursor.execute("ALTER TABLE core_tag_new RENAME TO core_tag;")
 
@@ -491,6 +555,15 @@ def upgrade_core_tables(apps, schema_editor):
         print("    ✓ Core table rebuild complete")
 
 
+def _pg_sync_schema(apps, schema_editor):
+    # On postgres, upgrade_core_tables is a no-op above, so the raw sqlite
+    # rebuilds of core_tag/core_snapshot/core_archiveresult never ran. Resync
+    # the real schema to this migration's end-state (tables are empty on pg).
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["Tag", "Snapshot", "SnapshotTag", "ArchiveResult"])
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("core", "0022_auto_20231023_2008"),
@@ -571,4 +644,5 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0024_assign_default_crawl.py

@@ -4,94 +4,9 @@
 from django.db import migrations, models
 
 
-def create_default_crawl_and_assign_snapshots(apps, schema_editor):
-    """
-    Create a default crawl for migrated snapshots and assign all snapshots without a crawl to it.
-    Uses raw SQL because the app registry isn't fully populated during migrations.
-    """
-    from django.db import connection
-    import uuid as uuid_lib
-    from datetime import datetime
-
-    cursor = connection.cursor()
-
-    # Check if there are any snapshots without a crawl
-    cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id IS NULL")
-    snapshots_without_crawl = cursor.fetchone()[0]
-
-    if snapshots_without_crawl == 0:
-        print("✓ Fresh install or all snapshots already have crawls")
-        return
-
-    cursor.execute("SELECT url FROM core_snapshot WHERE crawl_id IS NULL ORDER BY bookmarked_at, timestamp")
-    crawl_urls = "\n".join(url for (url,) in cursor.fetchall() if url)
-
-    # Get or create system user (pk=1)
-    cursor.execute("SELECT id FROM auth_user WHERE id = 1")
-    if not cursor.fetchone():
-        cursor.execute(
-            """
-            INSERT INTO auth_user (id, password, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)
-            VALUES (1, '!', 1, 'system', '', '', '', 1, 1, ?)
-        """,
-            [datetime.now().isoformat()],
-        )
-
-    # Create a default crawl for migrated snapshots.
-    # Depending on migration graph order, later crawls migrations may already
-    # have removed output_dir by the time this data migration runs.
-    crawl_id = uuid_lib.uuid4().hex
-    now = datetime.now().isoformat()
-    cursor.execute("PRAGMA table_info(crawls_crawl)")
-    crawl_columns = {row[1] for row in cursor.fetchall()}
-    default_columns: list[str] = []
-    default_values: list[str] = []
-    if "output_dir" in crawl_columns:
-        default_columns.append("output_dir")
-        default_values.append("''")
-    for column in ("max_size", "max_urls", "crawl_max_size", "snapshot_max_size"):
-        if column in crawl_columns:
-            default_columns.append(column)
-            default_values.append("0")
-    default_columns_sql = f"{', '.join(default_columns)}, " if default_columns else ""
-    default_values_sql = f"{', '.join(default_values)}, " if default_values else ""
-
-    cursor.execute(
-        f"""
-        INSERT INTO crawls_crawl (
-            id, created_at, modified_at, num_uses_succeeded, num_uses_failed,
-            urls, max_depth, tags_str, label, notes, {default_columns_sql}
-            status, retry_at, created_by_id, schedule_id, config, persona_id
-        ) VALUES (?, ?, ?, 0, 0, ?, 0, '', 'Migrated from v0.7.2/v0.8.6',
-                  'Auto-created crawl for migrated snapshots', {default_values_sql}
-                  'sealed', ?, 1, NULL, '{{}}', NULL)
-    """,
-        [crawl_id, now, now, crawl_urls, now],
-    )
-
-    # Assign all snapshots without a crawl to the default crawl
-    cursor.execute("UPDATE core_snapshot SET crawl_id = ? WHERE crawl_id IS NULL", [crawl_id])
-
-    print(f"✓ Assigned {snapshots_without_crawl} snapshots to default crawl {crawl_id}")
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("core", "0023_upgrade_to_0_9_0"),
-        ("crawls", "0002_upgrade_from_0_8_6"),
-        ("auth", "0012_alter_user_first_name_max_length"),
-    ]
-
-    operations = [
-        migrations.RunPython(
-            create_default_crawl_and_assign_snapshots,
-            reverse_code=migrations.RunPython.noop,
-        ),
-        migrations.SeparateDatabaseAndState(
-            database_operations=[
-                # Now make crawl_id NOT NULL
-                migrations.RunSQL(
-                    sql="""
+# Raw sqlite table-rebuild that makes core_snapshot.crawl_id NOT NULL. Kept
+# byte-for-byte and replayed through Django's own RunSQL only on sqlite.
+_MAKE_CRAWL_ID_NOT_NULL_SQL = """
                         -- Rebuild snapshot table with NOT NULL crawl_id
                         CREATE TABLE core_snapshot_final (
                             id TEXT PRIMARY KEY NOT NULL,
@@ -153,8 +68,122 @@ class Migration(migrations.Migration):
                         CREATE INDEX core_snapshot_retry_at_idx ON core_snapshot(retry_at);
                         CREATE INDEX core_snapshot_created_at_idx ON core_snapshot(created_at);
                         CREATE UNIQUE INDEX core_snapshot_url_crawl_unique ON core_snapshot(url, crawl_id);
-                    """,
-                    reverse_sql=migrations.RunSQL.noop,
+                    """
+
+
+def _make_crawl_id_not_null(apps, schema_editor):
+    # sqlite-only table rebuild. Reuse Django's own RunSQL statement splitting so
+    # the sqlite behavior is byte-for-byte identical to the original RunSQL op.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+    migrations.RunSQL(sql=_MAKE_CRAWL_ID_NOT_NULL_SQL).database_forwards("core", schema_editor, None, None)
+
+
+def _pg_sync_schema(apps, schema_editor):
+    # On postgres the raw sqlite rebuilds above are skipped and crawl was only
+    # added to migration state; resync the real schema to this migration's
+    # end-state (empty tables). Snapshot is referenced by SnapshotTag and
+    # ArchiveResult, so those are rebuilt too to restore FK constraints.
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["Snapshot", "SnapshotTag", "ArchiveResult"])
+
+
+def create_default_crawl_and_assign_snapshots(apps, schema_editor):
+    """
+    Create a default crawl for migrated snapshots and assign all snapshots without a crawl to it.
+    Uses raw SQL because the app registry isn't fully populated during migrations.
+    """
+    # Legacy-data-only and sqlite-specific because it uses PRAGMA introspection.
+    # postgres install core_snapshot has no crawl_id column yet and there are no
+    # unassigned snapshots, so gate before touching any SQL.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
+    from django.db import connection
+    import uuid as uuid_lib
+    from datetime import datetime
+
+    cursor = connection.cursor()
+
+    # Check if there are any snapshots without a crawl
+    cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id IS NULL")
+    snapshots_without_crawl = cursor.fetchone()[0]
+
+    if snapshots_without_crawl == 0:
+        print("✓ Fresh install or all snapshots already have crawls")
+        return
+
+    cursor.execute("SELECT url FROM core_snapshot WHERE crawl_id IS NULL ORDER BY bookmarked_at, timestamp")
+    crawl_urls = "\n".join(url for (url,) in cursor.fetchall() if url)
+
+    # Get or create system user (pk=1)
+    cursor.execute("SELECT id FROM auth_user WHERE id = 1")
+    if not cursor.fetchone():
+        cursor.execute(
+            """
+            INSERT INTO auth_user (id, password, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)
+            VALUES (1, '!', 1, 'system', '', '', '', 1, 1, %s)
+        """,
+            [datetime.now().isoformat()],
+        )
+
+    # Create a default crawl for migrated snapshots.
+    # Depending on migration graph order, later crawls migrations may already
+    # have removed output_dir by the time this data migration runs.
+    crawl_id = uuid_lib.uuid4().hex
+    now = datetime.now().isoformat()
+    cursor.execute("PRAGMA table_info(crawls_crawl)")
+    crawl_columns = {row[1] for row in cursor.fetchall()}
+    default_columns: list[str] = []
+    default_values: list[str] = []
+    if "output_dir" in crawl_columns:
+        default_columns.append("output_dir")
+        default_values.append("''")
+    for column in ("max_size", "max_urls", "crawl_max_size", "snapshot_max_size"):
+        if column in crawl_columns:
+            default_columns.append(column)
+            default_values.append("0")
+    default_columns_sql = f"{', '.join(default_columns)}, " if default_columns else ""
+    default_values_sql = f"{', '.join(default_values)}, " if default_values else ""
+
+    cursor.execute(
+        f"""
+        INSERT INTO crawls_crawl (
+            id, created_at, modified_at, num_uses_succeeded, num_uses_failed,
+            urls, max_depth, tags_str, label, notes, {default_columns_sql}
+            status, retry_at, created_by_id, schedule_id, config, persona_id
+        ) VALUES (%s, %s, %s, 0, 0, %s, 0, '', 'Migrated from v0.7.2/v0.8.6',
+                  'Auto-created crawl for migrated snapshots', {default_values_sql}
+                  'sealed', %s, 1, NULL, '{{}}', NULL)
+    """,
+        [crawl_id, now, now, crawl_urls, now],
+    )
+
+    # Assign all snapshots without a crawl to the default crawl
+    cursor.execute("UPDATE core_snapshot SET crawl_id = %s WHERE crawl_id IS NULL", [crawl_id])
+
+    print(f"✓ Assigned {snapshots_without_crawl} snapshots to default crawl {crawl_id}")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0023_upgrade_to_0_9_0"),
+        ("crawls", "0002_upgrade_from_0_8_6"),
+        ("auth", "0012_alter_user_first_name_max_length"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            create_default_crawl_and_assign_snapshots,
+            reverse_code=migrations.RunPython.noop,
+        ),
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                # Now make crawl_id NOT NULL (sqlite-only table rebuild)
+                migrations.RunPython(
+                    _make_crawl_id_not_null,
+                    reverse_code=migrations.RunPython.noop,
                 ),
             ],
             state_operations=[
@@ -169,4 +198,5 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0025_alter_archiveresult_options_alter_snapshot_options_and_more.py

@@ -11,6 +11,12 @@ from archivebox.uuid_compat import uuid7
 
 def copy_old_fields_to_new(apps, schema_editor):
     """Copy data from old field names to new field names after AddField operations."""
+    # sqlite-only legacy data copy (PRAGMA introspection). On postgres there is
+    # no legacy data and the state-only AddFields below are resynced by
+    # _pg_sync_schema at the end of this migration.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
     cursor = connection.cursor()
 
     # Check if old fields still exist
@@ -43,6 +49,20 @@ def copy_old_fields_to_new(apps, schema_editor):
     # NOTE: UUIDs are already populated by migration 0023 for all migration paths.
 
 
+def _pg_sync_schema(apps, schema_editor):
+    # This migration mixes real ORM AddFields with state-only AddFields (the
+    # snapshot config/current_step/depth/notes/num_uses_*/parent_snapshot/
+    # retry_at/status columns and archiveresult created_at/modified_at were
+    # created on sqlite by 0023's raw rebuild, so here they are state-only).
+    # On postgres those columns would never land, desyncing the real schema.
+    # Resync to this migration's final state (empty tables on pg). Tag and
+    # Snapshot both gain state-only fields; Snapshot is referenced by
+    # SnapshotTag and ArchiveResult, so rebuild those too.
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["Tag", "Snapshot", "SnapshotTag", "ArchiveResult"])
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("core", "0024_assign_default_crawl"),
@@ -61,8 +81,8 @@ class Migration(migrations.Migration):
         ),
         # NOTE: RemoveField for cmd, cmd_version, pwd moved to migration 0027
         # to allow data migration to Process records first
-        # NOTE: created_at/modified_at are created by migration 0023 so they can
-        # preserve old ArchiveResult row metadata before SQLite table rebuilds.
+        # NOTE: created_at/modified_at/notes are created by migration 0023 so
+        # they can preserve old ArchiveResult metadata before SQLite rebuilds.
         # Update Django's state here before the AddField operations below.
         migrations.SeparateDatabaseAndState(
             database_operations=[],
@@ -77,6 +97,11 @@ class Migration(migrations.Migration):
                     name="modified_at",
                     field=models.DateTimeField(auto_now=True),
                 ),
+                migrations.AddField(
+                    model_name="archiveresult",
+                    name="notes",
+                    field=models.TextField(blank=True, default=""),
+                ),
             ],
         ),
         migrations.AddField(
@@ -95,11 +120,6 @@ class Migration(migrations.Migration):
                 max_length=255,
             ),
         ),
-        migrations.AddField(
-            model_name="archiveresult",
-            name="notes",
-            field=models.TextField(blank=True, default=""),
-        ),
         migrations.AddField(
             model_name="archiveresult",
             name="num_uses_failed",
@@ -218,26 +238,33 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
-        migrations.AddField(
-            model_name="tag",
-            name="created_at",
-            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, null=True),
-        ),
-        migrations.AddField(
-            model_name="tag",
-            name="created_by",
-            field=models.ForeignKey(
-                default=archivebox.base_models.models.get_or_create_system_user_pk,
-                null=True,
-                on_delete=django.db.models.deletion.CASCADE,
-                related_name="tag_set",
-                to=settings.AUTH_USER_MODEL,
-            ),
-        ),
-        migrations.AddField(
-            model_name="tag",
-            name="modified_at",
-            field=models.DateTimeField(auto_now=True),
+        # 0023 creates these columns before rebuilding the legacy Tag table so
+        # their values survive. Declare them in state without rebuilding it.
+        migrations.SeparateDatabaseAndState(
+            database_operations=[],
+            state_operations=[
+                migrations.AddField(
+                    model_name="tag",
+                    name="created_at",
+                    field=models.DateTimeField(db_index=True, default=django.utils.timezone.now, null=True),
+                ),
+                migrations.AddField(
+                    model_name="tag",
+                    name="created_by",
+                    field=models.ForeignKey(
+                        default=archivebox.base_models.models.get_or_create_system_user_pk,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="tag_set",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+                migrations.AddField(
+                    model_name="tag",
+                    name="modified_at",
+                    field=models.DateTimeField(auto_now=True),
+                ),
+            ],
         ),
         # Copy data from old field names to new field names after AddField operations
         migrations.RunPython(
@@ -335,4 +362,5 @@ class Migration(migrations.Migration):
             model_name="snapshot",
             constraint=models.UniqueConstraint(fields=("timestamp",), name="unique_timestamp"),
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0027_copy_archiveresult_to_process.py

@@ -1,7 +1,7 @@
 # Generated by hand on 2026-01-01
 # Copies ArchiveResult cmd/pwd/cmd_version data to Process records before removing old fields
 
-from django.db import migrations, connection
+from django.db import migrations
 import json
 from pathlib import Path
 from archivebox.uuid_compat import uuid7
@@ -51,7 +51,7 @@ def get_or_create_current_machine(cursor):
     guid = f"host_{hostname}"  # Simple but stable identifier
 
     # Check if machine exists
-    cursor.execute("SELECT id FROM machine_machine WHERE guid = ?", [guid])
+    cursor.execute("SELECT id FROM machine_machine WHERE guid = %s", [guid])
     row = cursor.fetchone()
 
     if row:
@@ -76,7 +76,7 @@ def get_or_create_current_machine(cursor):
                 hw_in_docker, hw_in_vm, hw_manufacturer, hw_product, hw_uuid,
                 os_arch, os_family, os_platform, os_release, os_kernel,
                 stats, config, num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 0, 0, '', '', '',
+            ) VALUES (%s, %s, %s, %s, %s, 0, 0, '', '', '',
                       '', '', '', '', '', '{}', '{}', 0, 0)
         """,
             [machine_id, now, now, guid, hostname],
@@ -90,7 +90,7 @@ def get_or_create_current_machine(cursor):
                 hw_in_docker, hw_in_vm, hw_manufacturer, hw_product, hw_uuid,
                 os_arch, os_family, os_platform, os_release, os_kernel,
                 stats, num_uses_failed, num_uses_succeeded
-            ) VALUES (?, ?, ?, ?, ?, 0, 0, '', '', '',
+            ) VALUES (%s, %s, %s, %s, %s, 0, 0, '', '', '',
                       '', '', '', '', '', '{}', 0, 0)
         """,
             [machine_id, now, now, guid, hostname],
@@ -125,7 +125,7 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
     cursor.execute(
         """
         SELECT id FROM machine_binary
-        WHERE machine_id = ? AND name = ? AND abspath = ? AND version = ?
+        WHERE machine_id = %s AND name = %s AND abspath = %s AND version = %s
     """,
         [machine_id, name, abspath, version],
     )
@@ -162,7 +162,7 @@ def get_or_create_binary(cursor, machine_id, name, abspath, version):
         "num_uses_succeeded": 0,
     }
     insert_cols = [col for col in values_by_col if col in binary_cols]
-    placeholders = ", ".join(["?"] * len(insert_cols))
+    placeholders = ", ".join(["%s"] * len(insert_cols))
     cursor.execute(
         f"""
         INSERT INTO machine_binary ({", ".join(insert_cols)})
@@ -244,7 +244,7 @@ def create_process(cursor, machine_id, pwd, cmd, status, exit_code, started_at,
         "num_uses_succeeded": 0,
     }
     insert_cols = [col for col in values_by_col if col in process_cols]
-    placeholders = ", ".join(["?"] * len(insert_cols))
+    placeholders = ", ".join(["%s"] * len(insert_cols))
     cursor.execute(
         f"""
         INSERT INTO machine_process ({", ".join(insert_cols)})
@@ -275,7 +275,15 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
     - failed → exited (exit_code=1)
     - skipped → exited (exit_code=None)
     """
-    cursor = connection.cursor()
+    # sqlite-only legacy data copy because it uses PRAGMA introspection.
+    # A postgres install can never contain legacy ArchiveResult cmd/pwd data at
+    # this point, so there is nothing to copy into machine_process. The
+    # RemoveField ops below are real (non-state-only) and drop the now-empty
+    # cmd/pwd/cmd_version columns on both vendors, so no resync is needed.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
+    cursor = schema_editor.connection.cursor()
 
     # Check if old fields still exist (skip if fresh install or already migrated)
     cursor.execute("PRAGMA table_info(core_archiveresult)")
@@ -287,96 +295,101 @@ def copy_archiveresult_data_to_process(apps, schema_editor):
 
     # Check if process_id field exists (should exist from 0026)
     if "process_id" not in cols:
-        print("✗ ERROR: process_id field not found. Migration 0026 must run first.")
-        return
+        raise RuntimeError("process_id field not found. Migration 0026 must run first.")
 
     # Get or create Machine.current()
     machine_id = get_or_create_current_machine(cursor)
 
-    # Get ArchiveResults without process_id that have cmd data
+    # Get every ArchiveResult without a Process. Rows with empty execution
+    # metadata still need a Process before the legacy columns can be removed.
     # Use plugin (extractor was renamed to plugin in migration 0025)
     cursor.execute("""
         SELECT id, snapshot_id, plugin, cmd, pwd, cmd_version,
                status, start_ts, end_ts, created_at
         FROM core_archiveresult
         WHERE process_id IS NULL
-        AND (cmd IS NOT NULL OR pwd IS NOT NULL)
     """)
 
     results = cursor.fetchall()
 
     if not results:
         print("    ✓ No ArchiveResults need Process migration")
-        return
-
-    print(f"    - Migrating {len(results)} ArchiveResults to Process rows...")
+    else:
+        print(f"    - Migrating {len(results)} ArchiveResults to Process rows...")
 
     migrated_count = 0
-    skipped_count = 0
-    error_count = 0
-
     for i, row in enumerate(results):
         ar_id, snapshot_id, plugin, cmd_raw, pwd, cmd_version, status, start_ts, end_ts, created_at = row
 
-        try:
-            # Parse cmd field
-            cmd_array = parse_cmd_field(cmd_raw)
+        # Parse cmd field
+        cmd_array = parse_cmd_field(cmd_raw)
 
-            # Extract binary info from cmd[0] if available
-            binary_id = None
-            if cmd_array and cmd_array[0]:
-                binary_name = Path(cmd_array[0]).name or plugin  # Fallback to plugin name
-                binary_abspath = cmd_array[0]
-                binary_version = cmd_version or ""
+        # Extract binary info from cmd[0] if available
+        binary_id = None
+        if cmd_array and cmd_array[0]:
+            binary_name = Path(cmd_array[0]).name or plugin  # Fallback to plugin name
+            binary_abspath = cmd_array[0]
+            binary_version = cmd_version or ""
 
-                # Get or create Binary record
-                binary_id = get_or_create_binary(
-                    cursor,
-                    machine_id,
-                    binary_name,
-                    binary_abspath,
-                    binary_version,
-                )
-
-            # Map status
-            process_status, exit_code = map_status(status)
-
-            # Set timestamps
-            started_at = start_ts or created_at
-            ended_at = end_ts if process_status == "exited" else None
-
-            # Create Process record
-            process_id = create_process(
-                cursor=cursor,
-                machine_id=machine_id,
-                pwd=pwd or "",
-                cmd=cmd_array,
-                status=process_status,
-                exit_code=exit_code,
-                started_at=started_at,
-                ended_at=ended_at,
-                binary_id=binary_id,
+            # Get or create Binary record
+            binary_id = get_or_create_binary(
+                cursor,
+                machine_id,
+                binary_name,
+                binary_abspath,
+                binary_version,
             )
+        elif cmd_version:
+            raise RuntimeError(f"ArchiveResult {ar_id} has cmd_version metadata but no command")
 
-            # Link ArchiveResult to Process
-            cursor.execute(
-                "UPDATE core_archiveresult SET process_id = ? WHERE id = ?",
-                [process_id, ar_id],
-            )
+        # Map status
+        process_status, exit_code = map_status(status)
 
-            migrated_count += 1
-            if migrated_count % PROGRESS_EVERY == 0:
-                print(f"      migrated {migrated_count}/{len(results)} ArchiveResults...")
+        # Set timestamps
+        started_at = start_ts or created_at
+        ended_at = end_ts if process_status == "exited" else None
 
-        except Exception as e:
-            print(f"✗ Error migrating ArchiveResult {ar_id}: {e}")
-            import traceback
+        # Create Process record
+        process_id = create_process(
+            cursor=cursor,
+            machine_id=machine_id,
+            pwd=pwd or "",
+            cmd=cmd_array,
+            status=process_status,
+            exit_code=exit_code,
+            started_at=started_at,
+            ended_at=ended_at,
+            binary_id=binary_id,
+        )
 
-            traceback.print_exc()
-            error_count += 1
-            continue
+        # Link ArchiveResult to Process
+        cursor.execute(
+            "UPDATE core_archiveresult SET process_id = %s WHERE id = %s",
+            [process_id, ar_id],
+        )
+        if cursor.rowcount != 1:
+            raise RuntimeError(f"ArchiveResult {ar_id} was not linked to Process {process_id}")
 
-    print(f"    ✓ Process migration complete: {migrated_count} migrated, {skipped_count} skipped, {error_count} errors")
+        migrated_count += 1
+        if migrated_count % PROGRESS_EVERY == 0:
+            print(f"      migrated {migrated_count}/{len(results)} ArchiveResults...")
+
+    cursor.execute("""
+        SELECT ar.id, ar.cmd, ar.pwd, ar.cmd_version, ar.process_id,
+               process.cmd, process.pwd, binary.version
+        FROM core_archiveresult AS ar
+        LEFT JOIN machine_process AS process ON process.id = ar.process_id
+        LEFT JOIN machine_binary AS binary ON binary.id = process.binary_id
+    """)
+    for ar_id, cmd_raw, pwd, cmd_version, process_id, process_cmd_raw, process_pwd, binary_version in cursor.fetchall():
+        if not process_id:
+            raise RuntimeError(f"ArchiveResult {ar_id} is not linked to a Process")
+        if parse_cmd_field(process_cmd_raw) != parse_cmd_field(cmd_raw) or process_pwd != (pwd or ""):
+            raise RuntimeError(f"ArchiveResult {ar_id} execution metadata was not preserved")
+        if cmd_version and binary_version != cmd_version:
+            raise RuntimeError(f"ArchiveResult {ar_id} command version was not preserved")
+
+    print(f"    ✓ Process migration complete: {migrated_count} migrated")
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0029_migrate_archiveresult_to_uuid_pk.py

@@ -1,7 +1,7 @@
 # Generated by hand on 2026-01-02
 # Migrate ArchiveResult from integer PK to UUID PK (matching Snapshot)
 
-from django.db import migrations, models, connection
+from django.db import migrations, models
 from uuid import UUID
 from archivebox.uuid_compat import uuid7
 
@@ -26,7 +26,13 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
 
     Result: Clean schema with ONLY id as UUIDField (no old_id, no uuid)
     """
-    cursor = connection.cursor()
+    # sqlite-only table rebuild (sqlite_master, PRAGMA, table copy/rename). On
+    # postgres this is unnecessary and invalid: the table is empty and the
+    # state flip of id -> UUIDField is applied by _pg_sync_schema below.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
+    cursor = schema_editor.connection.cursor()
 
     # Check if table exists and has data
     cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'")
@@ -95,23 +101,35 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     col_names = [col[1] for col in columns]
     has_uuid_column = "uuid" in col_names
 
+    used_uuids = set()
+
+    def fresh_uuid():
+        while True:
+            candidate = uuid7().hex
+            if candidate not in used_uuids:
+                used_uuids.add(candidate)
+                return candidate
+
+    def unique_uuid(existing_uuid):
+        try:
+            candidate = UUID(str(existing_uuid)).hex if existing_uuid else None
+        except (TypeError, ValueError, AttributeError):
+            candidate = None
+
+        if candidate and candidate not in used_uuids:
+            used_uuids.add(candidate)
+            return candidate
+        return fresh_uuid()
+
     if has_uuid_column:
         cursor.execute("SELECT id, uuid FROM core_archiveresult")
         records = cursor.fetchall()
-        id_to_uuid = {}
-        for old_id, existing_uuid in records:
-            if existing_uuid:
-                # Normalize existing UUID to 32-char hex format (Django SQLite UUIDField format)
-                # (existing UUIDs might be stored with or without dashes in old schema)
-                id_to_uuid[old_id] = UUID(existing_uuid).hex
-            else:
-                # Generate new UUIDv7 (time-ordered) as 32-char hex
-                id_to_uuid[old_id] = uuid7().hex
+        id_to_uuid = {old_id: unique_uuid(existing_uuid) for old_id, existing_uuid in records}
     else:
         # 0.7.x path: no uuid column, generate new UUIDs for all records
         cursor.execute("SELECT id FROM core_archiveresult")
         records = cursor.fetchall()
-        id_to_uuid = {old_id: uuid7().hex for (old_id,) in records}
+        id_to_uuid = {old_id: fresh_uuid() for (old_id,) in records}
 
     # Step 3: Copy data with UUIDs as new primary key
     cursor.execute("SELECT * FROM core_archiveresult")
@@ -119,7 +137,7 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
 
     # col_names already fetched in Step 2
     inserted_count = 0
-    for i, record in enumerate(old_records):
+    for record in old_records:
         old_id = record[col_names.index("id")]
         new_uuid = id_to_uuid[old_id]
 
@@ -152,29 +170,34 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
         # Build INSERT statement (only copy fields that exist in source)
         existing_fields = [f for f in fields_to_copy if f in values]
 
-        placeholders = ", ".join(["?"] * (len(existing_fields) + 1))  # +1 for id
+        placeholders = ", ".join(["%s"] * (len(existing_fields) + 1))  # +1 for id
         field_list = "id, " + ", ".join(existing_fields)
 
         insert_values = [new_uuid] + [values.get(f) for f in existing_fields]
 
-        try:
-            cursor.execute(
-                f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})",
-                insert_values,
-            )
-            inserted_count += 1
-            if inserted_count % PROGRESS_EVERY == 0:
-                print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
-        except Exception as e:
-            print(f"[0029] ERROR inserting record {old_id}: {e}")
-            if i == 0:
-                print(f"[0029] First record values: {insert_values[:5]}...")
-                raise
+        cursor.execute(
+            f"INSERT INTO core_archiveresult_new ({field_list}) VALUES ({placeholders})",
+            insert_values,
+        )
+        inserted_count += 1
+        if inserted_count % PROGRESS_EVERY == 0:
+            print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults...")
 
     if old_records:
         print(f"      copied {inserted_count}/{len(old_records)} ArchiveResults")
 
-    # Step 4: Replace old table with new table
+    cursor.execute("SELECT COUNT(*) FROM core_archiveresult")
+    source_count = cursor.fetchone()[0]
+    cursor.execute("SELECT COUNT(*) FROM core_archiveresult_new")
+    destination_count = cursor.fetchone()[0]
+    if source_count != row_count or destination_count != row_count or inserted_count != row_count:
+        raise RuntimeError(
+            "ArchiveResult UUID migration row-count mismatch: "
+            f"expected={row_count}, source={source_count}, destination={destination_count}, inserted={inserted_count}",
+        )
+
+    # Step 4: Replace old table with new table. Django runs this migration in
+    # one transaction, so any failure above leaves the source table untouched.
     cursor.execute("DROP TABLE core_archiveresult")
     cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult")
 
@@ -190,6 +213,16 @@ def migrate_archiveresult_id_to_uuid(apps, schema_editor):
     print(f"    ✓ ArchiveResult UUID primary key migration complete ({row_count} records)")
 
 
+def _pg_sync_schema(apps, schema_editor):
+    # On postgres the sqlite table rebuild above is skipped; the id -> UUIDField
+    # flip and uuid-field removal only reach migration state. Resync the real
+    # (empty) core_archiveresult table to this migration's end-state. Nothing in
+    # core references ArchiveResult, so no other models need rebuilding.
+    from archivebox.misc.db import rebuild_models_from_migration_state
+
+    rebuild_models_from_migration_state(apps, schema_editor, "core", ["ArchiveResult"])
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("core", "0028_alter_snapshot_fs_version"),
@@ -217,4 +250,5 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.RunPython(_pg_sync_schema, reverse_code=migrations.RunPython.noop),
     ]

archivebox/core/migrations/0045_archiveresult_unique_hook.py

@@ -2,13 +2,13 @@ from django.db import migrations, models
 
 
 def deduplicate_archiveresults_per_hook(apps, schema_editor):
-    """Drop duplicate ArchiveResult rows per (snapshot, plugin, hook_name).
+    """Give historical duplicate rows unique hook identities.
 
     Real long-lived collections (cabbage's demo, beta-tester DBs) accumulated
-    multiple rows per hook over the dev rc chain. The next operation adds a
-    UniqueConstraint on that tuple; without this cleanup pass the constraint
-    fails with ``UNIQUE constraint failed`` mid-migration and bricks startup.
-    Keep the row with the highest id (most recent) for each tuple.
+    multiple rows per hook. Preserve every row while keeping the newest row on
+    the canonical hook identity expected by the current scheduler. Older rows
+    receive deterministic historical identities before the existing unique
+    constraint is added.
     """
     ArchiveResult = apps.get_model("core", "ArchiveResult")
     duplicate_groups = (
@@ -20,9 +20,10 @@ def deduplicate_archiveresults_per_hook(apps, schema_editor):
             "plugin": group["plugin"],
             "hook_name": group["hook_name"],
         }
-        keep = ArchiveResult.objects.filter(**lookup).order_by("-id").values_list("id", flat=True).first()
-        if keep is not None:
-            ArchiveResult.objects.filter(**lookup).exclude(id=keep).delete()
+        rows = list(ArchiveResult.objects.filter(**lookup).order_by("-id").values_list("id", flat=True))
+        for row_id in rows[1:]:
+            historical_name = f"__legacy_history__{row_id}"[:255]
+            ArchiveResult.objects.filter(id=row_id).update(hook_name=historical_name)
 
 
 class Migration(migrations.Migration):

archivebox/core/migrations/0046_repair_snapshot_permissions.py

@@ -13,6 +13,13 @@ def _repair_snapshot_permissions(apps, schema_editor):
     no underlying column. Fresh installs added the column via 0041 and
     this is a no-op.
     """
+    # sqlite-only legacy repair (PRAGMA table_xinfo + sqlite VIRTUAL generated
+    # column syntax). On postgres the permissions column was created by 0041's
+    # portable GeneratedField AddField (STORED generated column), so there is
+    # nothing to repair.
+    if schema_editor.connection.vendor != "sqlite":
+        return
+
     cursor = schema_editor.connection.cursor()
     # ``table_xinfo`` lists STORED/VIRTUAL generated columns; ``table_info``
     # silently drops them, so a prior 0041 that landed the STORED column

archivebox/core/migrations/0050_snapshot_permissions_not_null.py

@@ -0,0 +1,69 @@
+import django.db.models.fields.json
+import django.db.models.functions.comparison
+from django.db import migrations, models
+from django.db.models import Q
+
+
+VALID_PERMISSIONS = {"public", "unlisted", "private"}
+BATCH_SIZE = 1000
+
+
+def normalize_permissions(value, default="private"):
+    value = str(value or "").strip().lower()
+    return value if value in VALID_PERMISSIONS else default
+
+
+def hydrate_snapshot_permissions(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    db_alias = schema_editor.connection.alias
+    missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
+    batch = []
+
+    snapshots = (
+        Snapshot.objects.using(db_alias)
+        .filter(missing_permissions)
+        .select_related("crawl")
+        .only("id", "config", "crawl__permissions")
+        .iterator(chunk_size=BATCH_SIZE)
+    )
+    for snapshot in snapshots:
+        config = dict(snapshot.config or {})
+        config["PERMISSIONS"] = normalize_permissions(snapshot.crawl.permissions)
+        snapshot.config = config
+        batch.append(snapshot)
+        if len(batch) >= BATCH_SIZE:
+            Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+            batch.clear()
+
+    if batch:
+        Snapshot.objects.using(db_alias).bulk_update(batch, ["config"], batch_size=BATCH_SIZE)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("crawls", "0016_hydrate_crawl_permissions"),
+        ("core", "0049_alter_snapshot_url"),
+    ]
+
+    operations = [
+        migrations.RunPython(hydrate_snapshot_permissions, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="snapshot",
+            name="permissions",
+        ),
+        migrations.AddField(
+            model_name="snapshot",
+            name="permissions",
+            field=models.GeneratedField(
+                db_index=True,
+                db_persist=True,
+                editable=False,
+                expression=django.db.models.functions.comparison.Coalesce(
+                    django.db.models.fields.json.KeyTextTransform("PERMISSIONS", "config"),
+                    models.Value("private"),
+                    output_field=models.CharField(max_length=16),
+                ),
+                output_field=models.CharField(max_length=16),
+            ),
+        ),
+    ]

archivebox/core/migrations/0051_postgres_url_pattern_ops_index.py

@@ -0,0 +1,30 @@
+# Postgres-only: btree pattern-ops index on core_snapshot.url so LIKE 'prefix%'
+# queries (url__startswith and the URL prefix search) stay index scans under any
+# database collation. SQLite needs nothing here: its plain url index already
+# serves the bytewise range comparisons used on that backend.
+
+from django.db import migrations
+
+
+def add_pg_url_pattern_index(apps, schema_editor):
+    if schema_editor.connection.vendor != "postgresql":
+        return
+    schema_editor.execute(
+        "CREATE INDEX IF NOT EXISTS core_snapshot_url_pattern_ops_idx ON core_snapshot (url text_pattern_ops)",
+    )
+
+
+def remove_pg_url_pattern_index(apps, schema_editor):
+    if schema_editor.connection.vendor != "postgresql":
+        return
+    schema_editor.execute("DROP INDEX IF EXISTS core_snapshot_url_pattern_ops_idx")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0050_snapshot_permissions_not_null"),
+    ]
+
+    operations = [
+        migrations.RunPython(add_pg_url_pattern_index, remove_pg_url_pattern_index),
+    ]

archivebox/core/permissions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from django.db.models import Q, QuerySet
+from django.db.models import QuerySet
 from django.http import HttpRequest
 
 PERMISSIONS_PUBLIC = "public"
@@ -32,10 +32,7 @@ def is_admin_user(request: HttpRequest) -> bool:
 
 
 def get_snapshot_permissions(snapshot) -> str:
-    permissions = snapshot.permissions
-    if not permissions:
-        permissions = snapshot.crawl.permissions
-    return normalize_permissions(permissions)
+    return normalize_permissions(snapshot.permissions)
 
 
 def can_view_snapshot(request: HttpRequest, snapshot) -> bool:
@@ -56,10 +53,7 @@ def filter_personas_by_permissions(queryset: QuerySet, allowed_permissions: set[
 def filter_snapshots_by_permissions(queryset: QuerySet, *, direct: bool = False, allowed_permissions: set[str] | None = None) -> QuerySet:
     allowed_permissions = allowed_permissions or ({PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED} if direct else {PERMISSIONS_PUBLIC})
     allowed = sorted(allowed_permissions)
-    return queryset.exclude(
-        (Q(permissions__isnull=False) & ~Q(permissions__in=allowed))
-        | (Q(permissions__isnull=True) & (Q(crawl__permissions__isnull=True) | ~Q(crawl__permissions__in=allowed))),
-    )
+    return queryset.filter(permissions__in=allowed)
 
 
 def public_snapshots_queryset(queryset: QuerySet) -> QuerySet:

archivebox/core/preview_util.py

@@ -0,0 +1,4 @@
+__package__ = "archivebox.core"
+
+
+EXTENSION_SCREENSHOT_PLUGIN = "chrome_extension_screenshot"

archivebox/core/recovery_util.py

@@ -10,7 +10,7 @@ def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
     return exit_code is not None and (exit_code < 0 or exit_code >= 128)
 
 
-def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int]:
+def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.services.archive_result_service import _collect_output_metadata
@@ -21,10 +21,13 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
 
     now = timezone.now()
     recovery_console = Console(stderr=True, highlight=False, soft_wrap=True)
+    crawl_filter = {"id": crawl_id} if crawl_id else {}
+    snapshot_filter = {"crawl_id": crawl_id} if crawl_id else {}
+    result_filter = {"snapshot__crawl_id": crawl_id} if crawl_id else {}
     cleaned = {
-        "processes_stale_running": Process.cleanup_stale_running(),
-        "processes_orphaned_workers": Process.cleanup_orphaned_workers(),
-        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome else 0,
+        "processes_stale_running": 0 if crawl_id else Process.cleanup_stale_running(),
+        "processes_orphaned_workers": 0 if crawl_id else Process.cleanup_orphaned_workers(),
+        "chrome_processes_orphaned": Process.cleanup_orphaned_chrome() if include_chrome and not crawl_id else 0,
         "crawls_queued_without_retry_at": 0,
         "snapshots_queued_without_retry_at": 0,
         "archiveresults_backoff": 0,
@@ -58,14 +61,16 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     cleaned["crawls_queued_without_retry_at"] = Crawl.objects.filter(
         status=Crawl.StatusChoices.QUEUED,
         retry_at__isnull=True,
+        **crawl_filter,
     ).update(retry_at=now, modified_at=now)
     cleaned["snapshots_queued_without_retry_at"] = Snapshot.objects.filter(
         status=Snapshot.StatusChoices.QUEUED,
         retry_at__isnull=True,
         crawl__status__in=Crawl.RUNNABLE_STATES,
+        **snapshot_filter,
     ).update(retry_at=now, modified_at=now)
-    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF)
-    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED).exclude(
+    backoff_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.BACKOFF, **result_filter)
+    orphaned_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.STARTED, **result_filter).exclude(
         process__status=Process.StatusChoices.RUNNING,
     )
     # ArchiveResult has no retry_at scheduler. Wake only the parent Snapshots
@@ -92,7 +97,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     # only after this runner has won the single-runner gate, so it can safely
     # unlock those stale plugin leases for immediate processing instead of
     # waiting out the previous owner's full lock timeout.
-    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED)
+    queued_plugin_results = ArchiveResult.objects.filter(status=ArchiveResult.StatusChoices.QUEUED, **result_filter)
     cleaned["snapshots_queued_plugin_rows_waiting_on_stale_lease"] = (
         Snapshot.objects.filter(
             id__in=queued_plugin_results.values("snapshot_id"),
@@ -113,16 +118,33 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
         process_type=Process.TypeChoices.HOOK,
         archiveresult__isnull=True,
     ).exclude(status=Process.StatusChoices.RUNNING)
+    crawl_snapshot_ids = (
+        {str(snapshot_id) for snapshot_id in Snapshot.objects.filter(crawl_id=crawl_id).values_list("id", flat=True)} if crawl_id else None
+    )
+    if crawl_snapshot_ids is not None:
+        # Targeted `archivebox run --crawl-id ...` is used by foreground add/update
+        # commands and by resume of one existing crawl. It must still repair bad
+        # hook state for that crawl, but it must not scan historical orphaned hook
+        # rows from unrelated crawls before the first snapshot can run.
+        if crawl_snapshot_ids:
+            snapshot_pwd_filter = Q()
+            for snapshot_id in crawl_snapshot_ids:
+                snapshot_pwd_filter |= Q(pwd__contains=snapshot_id)
+            orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
+        else:
+            orphaned_hook_processes = orphaned_hook_processes.none()
     for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status"):
         hook_script_name = process.hook_script_name
         if not hook_script_name or not process.pwd:
             continue
         plugin_dir = Path(process.pwd)
+        if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
+            continue
         try:
             # Old or synthetic hook Process rows can point at arbitrary paths.
             # Only paths whose parent directory is a valid Snapshot id can be
             # reconstructed into ArchiveResult rows.
-            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name).first()
+            snapshot = Snapshot.objects.filter(id=plugin_dir.parent.name, **snapshot_filter).first()
         except ValidationError:
             continue
         if snapshot is None:
@@ -181,6 +203,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
             Snapshot.objects.filter(id=snapshot.id).update(retry_at=now, modified_at=now)
     started_snapshots = Snapshot.objects.filter(status=Snapshot.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **snapshot_filter,
     )
 
     # Broken lock repair: STARTED + retry_at=NULL or retry_at in the future
@@ -205,6 +228,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False) -> dict[str, int
     # are already final.
     recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+        **crawl_filter,
     )
 
     due_started_crawls = recoverable_started_crawls.annotate(has_due_child=Exists(due_child_snapshots)).filter(has_due_child=True)

archivebox/core/routes_util.py

@@ -4,12 +4,13 @@ import re
 from typing import Any
 from urllib.parse import urlparse
 
-from archivebox.config.common import get_config
+from django.utils.http import url_has_allowed_host_and_scheme
 
+from archivebox.config.common import get_config, get_request_config
 
 _SNAPSHOT_ID_RE = re.compile(r"^[0-9a-fA-F-]{8,36}$")
 _SNAPSHOT_SUBDOMAIN_RE = re.compile(r"^snap-(?P<suffix>[0-9a-fA-F]{12})$")
-_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api", "public")
+_ROLE_SUBDOMAIN_LABELS = ("admin", "web", "api")
 
 
 def split_host_port(host: str) -> tuple[str, str | None]:
@@ -102,7 +103,7 @@ def _with_port(host: str, port: str | None) -> str:
 
 
 def strip_role_subdomain(host: str) -> str:
-    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``public.`` / ``snap-*.``
+    """Strip leading ``admin.`` / ``web.`` / ``api.`` / ``snap-*.``
     labels from a host (preserving the port). Strips repeatedly so an
     already-compounded host like ``snap-X.snap-X.<base>`` reduces all the
     way down to ``<base>``.
@@ -151,7 +152,7 @@ def _root_host_from_listen(config: dict[str, Any] | None = None, **config_kwargs
 
 
 def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
     override = _normalize_base_url(config.BASE_URL)
     if override:
         return override
@@ -172,13 +173,12 @@ def get_base_url(request=None, config: dict[str, Any] | None = None, **config_kw
             return f"{scheme}://{_with_port('archivebox.localhost', req_port)}"
         # C) Per-request fallback: when ``BASE_URL`` is unset and CSRF didn't
         # give us a single origin, trust the request's Host header — but first
-        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``public.`` /
-        # ``snap-*.`` label. Otherwise the URL builders below prepend their own
-        # role label onto a host that already carries one, producing the
-        # ``snap-X.snap-X.snap-X.<base>`` compounding bug. Django has already
-        # admitted the host via ALLOWED_HOSTS; the misconfig banner surfaces
-        # the case where the resulting URL doesn't match what the operator
-        # probably intended.
+        # peel off any ``admin.`` / ``web.`` / ``api.`` / ``snap-*.`` label.
+        # Otherwise the URL builders below prepend their own role label onto a
+        # host that already carries one, producing the ``snap-X.snap-X.snap-X``
+        # compounding bug. Django has already admitted the host via
+        # ALLOWED_HOSTS; the misconfig banner surfaces the case where the
+        # resulting URL doesn't match what the operator probably intended.
         canonical_host = strip_role_subdomain(request.get_host())
         return f"{scheme}://{canonical_host}"
 
@@ -220,13 +220,6 @@ def get_api_host(config: dict[str, Any] | None = None, **config_kwargs: Any) ->
     return _build_base_host("api", config=config)
 
 
-def get_public_host(config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_host(config=config)
-    return _build_base_host("public", config=config)
-
-
 def get_snapshot_subdomain(snapshot_id: str) -> str:
     normalized = re.sub(r"[^0-9a-fA-F]", "", snapshot_id or "")
     suffix = (normalized[-12:] if len(normalized) >= 12 else normalized).lower()
@@ -287,9 +280,55 @@ def host_matches(request_host: str, target_host: str) -> bool:
     target_host_only, target_port = split_host_port(target_host)
     if req_host != target_host_only:
         return False
-    if target_port and req_port and target_port != req_port:
+    return not (target_port and req_port and target_port != req_port)
+
+
+def is_allowed_archivebox_redirect_url(url: str | None, request=None, config: dict[str, Any] | None = None) -> bool:
+    """Allow login redirects to this host or to the explicit ArchiveBox BASE_URL family.
+
+    Django already rejects arbitrary absolute ``next=`` URLs for the current host.
+    ArchiveBox also has legitimate cross-subdomain handoffs (admin/web/api/snap-*),
+    but only when BASE_URL is pinned. Without an explicit BASE_URL we do not widen
+    trust based on a request Host header that may be user supplied.
+    """
+    if not url:
         return False
-    return True
+
+    require_https = bool(request and request.is_secure())
+    request_host = request.get_host() if request else ""
+    if url_has_allowed_host_and_scheme(url=url, allowed_hosts={request_host}, require_https=require_https):
+        return True
+
+    parsed = urlparse(url)
+    if not parsed.netloc:
+        return False
+    if parsed.scheme not in {"http", "https"}:
+        return False
+    if require_https and parsed.scheme != "https":
+        return False
+    if parsed.username or parsed.password:
+        return False
+
+    config = config or get_config()
+    base_url = _normalize_base_url(config.BASE_URL)
+    if not base_url:
+        return False
+
+    base_host, base_port = split_host_port(urlparse(base_url).netloc)
+    if not base_host:
+        return False
+
+    redirect_host = (parsed.hostname or "").lower()
+    if redirect_host != base_host and not redirect_host.endswith(f".{base_host}"):
+        return False
+
+    try:
+        redirect_port = str(parsed.port) if parsed.port else None
+    except ValueError:
+        return False
+    if base_port:
+        return redirect_port == base_port
+    return redirect_port is None
 
 
 def _scheme_from_request(request=None, config: dict[str, Any] | None = None) -> str:
@@ -310,35 +349,28 @@ def _build_base_url_for_host(host: str, request=None, config: dict[str, Any] | N
 
 
 def get_admin_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
     if not config.USES_SUBDOMAIN_ROUTING:
         return get_base_url(request=request, config=config)
     return _build_base_url_for_host(_build_base_host("admin", request=request, config=config), request=request, config=config)
 
 
 def get_web_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
     if not config.USES_SUBDOMAIN_ROUTING:
         return get_base_url(request=request, config=config)
     return _build_base_url_for_host(_build_base_host("web", request=request, config=config), request=request, config=config)
 
 
 def get_api_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
     if not config.USES_SUBDOMAIN_ROUTING:
         return get_base_url(request=request, config=config)
     return _build_base_url_for_host(_build_base_host("api", request=request, config=config), request=request, config=config)
 
 
-def get_public_base_url(request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return get_base_url(request=request, config=config)
-    return _build_base_url_for_host(_build_base_host("public", request=request, config=config), request=request, config=config)
-
-
 def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
     if not config.USES_SUBDOMAIN_ROUTING:
         return _build_url(get_web_base_url(request=request, config=config), f"/snapshot/{str(snapshot_id).replace('-', '')}")
     return _build_base_url_for_host(
@@ -349,10 +381,8 @@ def get_snapshot_base_url(snapshot_id: str, request=None, config: dict[str, Any]
 
 
 def get_original_base_url(domain: str, request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:
-    config = config or get_config(**config_kwargs)
-    if not config.USES_SUBDOMAIN_ROUTING:
-        return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
-    return _build_base_url_for_host(_build_base_host(domain, request=request, config=config), request=request, config=config)
+    config = config or (get_request_config(request) if request is not None else get_config(**config_kwargs))
+    return _build_url(get_web_base_url(request=request, config=config), f"/original/{domain}")
 
 
 def build_admin_url(path: str = "", request=None, config: dict[str, Any] | None = None, **config_kwargs: Any) -> str:

archivebox/core/settings.py

@@ -1,28 +1,33 @@
 __package__ = "archivebox.core"
 
+import importlib
+import inspect
+import logging
 import os
 import sys
-import inspect
-import importlib
-
+from importlib.util import find_spec
 from pathlib import Path
 
 from django.conf.locale.en import formats as en_formats  # type: ignore
 
 import archivebox
-
-from archivebox.config.constants import CONSTANTS
 from archivebox.config.common import get_config
-from archivebox.core.routes_util import normalize_base_url, get_admin_base_url, get_api_base_url
+from archivebox.config.constants import CONSTANTS
+from archivebox.core.routes_util import get_admin_base_url, get_api_base_url, get_base_url, normalize_base_url
+
+# DATABASE_ENGINE config selects the backend (sqlite by default); the
+# sqlite-vs-postgres helpers live in archivebox.misc.db.
+from archivebox.misc.db import is_postgres, postgres_db_params
+
 from .settings_logging import SETTINGS_LOGGING
 
-
 IS_MIGRATING = "makemigrations" in sys.argv[:3] or "migrate" in sys.argv[:3]
-IS_TESTING = "test" in sys.argv[:3] or "PYTEST_CURRENT_TEST" in os.environ
+IS_TESTING = "test" in sys.argv[:3]
 IS_SHELL = "shell" in sys.argv[:3] or "shell_plus" in sys.argv[:3]
 IS_GETTING_VERSION_OR_HELP = "version" in sys.argv or "help" in sys.argv or "--version" in sys.argv or "--help" in sys.argv
 CONFIG = get_config()
 PACKAGE_DIR = CONSTANTS.PACKAGE_DIR
+logger = logging.getLogger(__name__)
 
 ################################################################################
 ### ArchiveBox Plugin Settings
@@ -72,7 +77,7 @@ INSTALLED_APPS = [
     "archivebox.crawls",  # handles Crawl and CrawlSchedule models and management (depends on core)
     "archivebox.progressmonitor",  # live progress endpoint and admin monitor template
     "archivebox.api",  # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
-    "abx_plugins.plugins.opencode",
+    "archivebox.opencode",
     # 3rd-party apps from PyPI that need to be loaded last
     "admin_data_views",  # handles rendering some convenient automatic read-only views of data in Django admin
     "django_extensions",  # provides Django Debug Toolbar (and other non-debug helpers)
@@ -84,6 +89,7 @@ DJANGO_OBJECT_ACTIONS_DEFAULT_HTTP_METHOD = "POST"
 MIDDLEWARE = [
     "archivebox.core.middleware.TimezoneMiddleware",
     "django.middleware.security.SecurityMiddleware",
+    "archivebox.core.middleware.AdminCookieIsolationMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
     "django.middleware.common.CommonMiddleware",
     "archivebox.api.middleware.ApiCorsMiddleware",
@@ -102,7 +108,6 @@ MIDDLEWARE = [
 ### Authentication Settings
 ################################################################################
 
-# AUTH_USER_MODEL = 'auth.User'   # cannot be easily changed unfortunately
 
 AUTHENTICATION_BACKENDS = [
     "django.contrib.auth.backends.RemoteUserBackend",
@@ -180,23 +185,11 @@ TEMPLATES_DIR_NAME = "templates"
 CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
 STATICFILES_DIRS = [
     *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
-    # *[
-    #     str(plugin_dir / 'static')
-    #     for plugin_dir in PLUGIN_DIRS.values()
-    #     if (plugin_dir / 'static').is_dir()
-    # ],
-    # Additional static file dirs from plugins
     str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "static"),
 ]
 
 TEMPLATE_DIRS = [
     *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
-    # *[
-    #     str(plugin_dir / 'templates')
-    #     for plugin_dir in PLUGIN_DIRS.values()
-    #     if (plugin_dir / 'templates').is_dir()
-    # ],
-    # Additional template dirs from plugins
     str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "core"),
     str(PACKAGE_DIR / TEMPLATES_DIR_NAME / "admin"),
     str(PACKAGE_DIR / TEMPLATES_DIR_NAME),
@@ -224,10 +217,6 @@ TEMPLATES = [
 ### External Service Settings
 ################################################################################
 
-# CACHE_DB_FILENAME = 'cache.sqlite3'
-# CACHE_DB_PATH = CONSTANTS.CACHE_DIR / CACHE_DB_FILENAME
-# CACHE_DB_TABLE = 'django_cache'
-
 DATABASE_NAME = CONFIG.DATABASE_NAME
 SQLITE_JOURNAL_MODE = CONFIG.SQLITE_JOURNAL_MODE
 SQLITE_MMAP_SIZE = CONFIG.SQLITE_MMAP_SIZE
@@ -261,20 +250,21 @@ SQLITE_CONNECTION_OPTIONS = {
     },
 }
 
-DATABASES = {
-    "default": {
-        "NAME": DATABASE_NAME,
-        **SQLITE_CONNECTION_OPTIONS,
-    },
-    # "filestore": {
-    #     "NAME": CONSTANTS.FILESTORE_DATABASE_FILE,
-    #     **SQLITE_CONNECTION_OPTIONS,
-    # },
-    # 'cache': {
-    #     'NAME': CACHE_DB_PATH,
-    #     **SQLITE_CONNECTION_OPTIONS,
-    # },
-}
+if is_postgres():
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.postgresql",
+            **postgres_db_params(),
+            "OPTIONS": {"connect_timeout": 10},
+        },
+    }
+else:
+    DATABASES = {
+        "default": {
+            "NAME": DATABASE_NAME,
+            **SQLITE_CONNECTION_OPTIONS,
+        },
+    }
 MIGRATION_MODULES = {"signal_webhooks": None}
 
 # Django requires DEFAULT_AUTO_FIELD to subclass AutoField (BigAutoField, SmallAutoField, etc.)
@@ -284,43 +274,10 @@ MIGRATION_MODULES = {"signal_webhooks": None}
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
 
-# class FilestoreDBRouter:
-#     """
-#     A router to store all the File models in the filestore.sqlite3 database.
-#     This data just mirrors what is in the file system, so we want to keep it in a separate database
-#     from the main index database to avoid contention.
-#     """
-
-#     route_app_labels = {"filestore"}
-#     db_name = "filestore"
-
-#     def db_for_read(self, model, **hints):
-#         if model._meta.app_label in self.route_app_labels:
-#             return self.db_name
-#         return 'default'
-
-#     def db_for_write(self, model, **hints):
-#         if model._meta.app_label in self.route_app_labels:
-#             return self.db_name
-#         return 'default'
-
-#     def allow_relation(self, obj1, obj2, **hints):
-#         if obj1._meta.app_label in self.route_app_labels or obj2._meta.app_label in self.route_app_labels:
-#             return obj1._meta.app_label == obj2._meta.app_label
-#         return None
-
-#     def allow_migrate(self, db, app_label, model_name=None, **hints):
-#         if app_label in self.route_app_labels:
-#             return db == self.db_name
-#         return db == "default"
-
 DATABASE_ROUTERS = []
 
 CACHES = {
     "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"},
-    # 'sqlite': {'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'cache'},
-    # 'dummy': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
-    # 'filebased': {"BACKEND": "django.core.cache.backends.filebased.FileBasedCache", "LOCATION": CACHE_DIR / 'cache_filebased'},
 }
 
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
@@ -378,20 +335,26 @@ try:
 
     _persisted_keys = _BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
     _secret_persisted = bool((_persisted_keys.get("SECRET_KEY") or "").strip())
-except Exception:
+except (OSError, RuntimeError, ValueError):
     _secret_persisted = True  # err on the side of NOT touching disk
 if not _secret_persisted:
     try:
         from archivebox.config.collection import write_config_file
 
         write_config_file({"SECRET_KEY": SECRET_KEY})
-    except Exception:
+    except (OSError, RuntimeError, ValueError):
         # Read-only mount, missing data dir, mid-init race — fall back to the
         # in-memory random key. The user will get logged out on the next boot
         # but the server still comes up.
-        pass
+        logger.debug("Unable to persist generated SECRET_KEY", exc_info=True)
 
 ALLOWED_HOSTS = [host.strip() for host in CONFIG.ALLOWED_HOSTS.split(",") if host.strip()]
+if (CONFIG.SERVER_SECURITY_MODE == "auto" or CONFIG.USES_SUBDOMAIN_ROUTING) and "*" not in ALLOWED_HOSTS:
+    # ArchiveBox owns the effective host policy at request time in auto and
+    # subdomain modes: canonical hosts keep the normal app surface, while
+    # alternate hosts are downgraded/rejected by get_request_config()/middleware.
+    # Django has to admit the Host header first for that existing boundary to run.
+    ALLOWED_HOSTS.append("*")
 CSRF_TRUSTED_ORIGINS = list({origin.strip() for origin in CONFIG.CSRF_TRUSTED_ORIGINS.split(",") if origin.strip()})
 
 admin_base_url = normalize_base_url(get_admin_base_url())
@@ -419,13 +382,28 @@ SECURE_BROWSER_XSS_FILTER = True
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
 
-CSRF_COOKIE_SECURE = False
-SESSION_COOKIE_SECURE = False
+# When BASE_URL is an https:// URL the deployment is HTTPS end-to-end, typically
+# behind a TLS-terminating proxy/tunnel (such as Cloudflare, Caddy, Traefik, or nginx)
+# where the proxy -> archivebox hop is plain HTTP, so
+# request.is_secure() / request.scheme would otherwise report http. Honour the
+# proxy's X-Forwarded-Proto so first-run URL detection and CSRF origin checks are
+# correct. Mark auth cookies Secure once the saved BASE_URL confirms HTTPS.
+# Derived from the RESOLVED base URL's scheme — no separate flag to keep in sync.
+# get_base_url() also covers deployments that only set CSRF_TRUSTED_ORIGINS (the
+# implicit-BASE_URL fallback used on 0.7.x->0.9.x upgrades), so HTTPS hardening
+# isn't lost until BASE_URL is migrated. An explicit plain-http base (e.g. local
+# http://archivebox.localhost:8000) disables proxy HTTPS handling.
+BASE_URL_IS_HTTPS = get_base_url(config=CONFIG).strip().lower().startswith("https://")
+if BASE_URL_IS_HTTPS or not CONFIG.BASE_URL:
+    SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+CSRF_COOKIE_SECURE = BASE_URL_IS_HTTPS
+SESSION_COOKIE_SECURE = BASE_URL_IS_HTTPS
 SESSION_COOKIE_HTTPONLY = True
 SESSION_COOKIE_NAME = f"archivebox_sessionid_{CONSTANTS.COLLECTION_ID}"
 CSRF_COOKIE_NAME = f"archivebox_csrftoken_{CONSTANTS.COLLECTION_ID}"
 # Auth cookies are intentionally scoped to the exact host that set them so
-# the admin session is NOT readable from public.* / web.* / api.* — that
+# the admin session is NOT readable from web.* / api.* — that
 # split is a security boundary, not a UX choice. Subdomains that need to
 # render an "is the user logged in?" indicator must use the single-bit
 # `archivebox_admin_logged_in` hint cookie set by core/middleware.py
@@ -626,8 +604,7 @@ if DEBUG:
 # https://github.com/bensi94/Django-Requests-Tracker (improved version of django-debug-toolbar)
 # Must delete archivebox/templates/admin to use because it relies on some things we override
 # visit /__requests_tracker__/ to access
-DEBUG_REQUESTS_TRACKER = True
-DEBUG_REQUESTS_TRACKER = DEBUG_REQUESTS_TRACKER and DEBUG
+DEBUG_REQUESTS_TRACKER = DEBUG and find_spec("requests_tracker") is not None
 if DEBUG_REQUESTS_TRACKER:
     import requests_tracker
 
@@ -650,16 +627,3 @@ if DEBUG_REQUESTS_TRACKER:
             r"^SELECT .* FROM django_migrations WHERE app = 'auth'",
         ),
     }
-
-# # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.)
-# DEBUG_LOGFIRE = False
-# DEBUG_LOGFIRE = DEBUG_LOGFIRE and os.access(DATA_DIR / '.logfire', os.W_OK) and (DATA_DIR / '.logfire').is_dir()
-
-
-# For usage with https://www.jetadmin.io/integrations/django
-# INSTALLED_APPS += ['jet_django']
-# JET_PROJECT = 'archivebox'
-# JET_TOKEN = 'some-api-token-here'
-
-
-# import ipdb; ipdb.set_trace()

archivebox/core/settings_logging.py

@@ -130,8 +130,6 @@ if os.access(LOGS_DIR, os.W_OK) and LOGS_DIR.is_dir():
     ERROR_LOG = LOGS_DIR / "errors.log"
 else:
     # historically too many edge cases here around creating log dir w/ correct permissions early on
-    # if there's an issue on startup, we trash the log and let user figure it out via stdout/stderr
-    # print(f'[!] WARNING: data/logs dir does not exist. Logging to temp file: {ERROR_LOG}')
     pass
 
 LOG_LEVEL_DATABASE = "WARNING"  # change to DEBUG to log all SQL queries

archivebox/core/setup_wizard.py

@@ -0,0 +1,75 @@
+from typing import Any
+
+from archivebox.core.routes_util import canonical_base_host_for_request, get_base_url, split_host_port
+
+
+def _request_scheme(request) -> str:
+    forwarded_scheme = (request.META.get("HTTP_X_FORWARDED_PROTO") or "").split(",", 1)[0].strip().lower()
+    if forwarded_scheme in {"http", "https"}:
+        return forwarded_scheme
+    return request.scheme or "http"
+
+
+def get_setup_wizard_context(request, config) -> dict[str, Any]:
+    """Build the first-run setup context when ``BASE_URL`` is unset."""
+    context = {
+        "mode": "unconfigured",
+        "canonical_host": "",
+        "display_host": "",
+        "suggested_base_url": "",
+        "machine_admin_url": "",
+        "can_configure": False,
+        "public_index": config.PUBLIC_INDEX,
+        "public_add_view": config.PUBLIC_ADD_VIEW,
+        "permissions": config.PERMISSIONS,
+    }
+    if request is None:
+        return context
+
+    scheme = _request_scheme(request)
+    canonical_host = canonical_base_host_for_request(request.get_host() or "")
+    display_hostname, display_port = split_host_port(canonical_host)
+    display_host = canonical_host
+    if (scheme, display_port) in (("http", "80"), ("https", "443")):
+        display_host = display_hostname
+
+    user = request.user
+    is_superuser = bool(user and user.is_authenticated and user.is_superuser)
+    machine_admin_url = ""
+    if is_superuser:
+        try:
+            from archivebox.machine.models import Machine
+
+            machine_admin_url = f"/admin/machine/machine/{Machine.current().id}/change/"
+        except (ImportError, RuntimeError, TypeError, ValueError):
+            machine_admin_url = ""
+
+    context.update(
+        canonical_host=canonical_host,
+        display_host=display_host,
+        suggested_base_url=f"{scheme}://{canonical_host}" if canonical_host else "",
+        machine_admin_url=machine_admin_url,
+        can_configure=is_superuser,
+    )
+    return context
+
+
+def get_base_url_mismatch_context(request, config) -> dict[str, str] | None:
+    """Describe a request origin that does not resolve to the configured base."""
+    if request is None or not config.BASE_URL:
+        return None
+
+    scheme = _request_scheme(request)
+    browser_url = f"{scheme}://{request.get_host()}".rstrip("/")
+    browser_base_url = browser_url
+    if config.USES_SUBDOMAIN_ROUTING:
+        browser_base_url = f"{scheme}://{canonical_base_host_for_request(request.get_host())}".rstrip("/")
+    configured_base_url = get_base_url(config=config).rstrip("/")
+    if browser_base_url.lower() == configured_base_url.lower():
+        return None
+
+    return {
+        "mode": "base_url_mismatch",
+        "browser_url": browser_url,
+        "configured_base_url": configured_base_url,
+    }

archivebox/core/tag_util.py

@@ -12,6 +12,7 @@ from django.http import HttpRequest
 from django.urls import reverse
 
 from archivebox.config.common import get_config
+from archivebox.misc.util import sanitize_html_text
 from archivebox.core.routes_util import build_snapshot_url, build_web_url
 from archivebox.core.models import Snapshot, SnapshotTag, Tag
 
@@ -33,7 +34,7 @@ TAG_HAS_SNAPSHOTS_CHOICES = (
 
 
 def normalize_tag_name(name: str) -> str:
-    return (name or "").strip()
+    return sanitize_html_text(name).strip()
 
 
 def normalize_tag_sort(sort: str = "created_desc") -> str:

archivebox/core/takeover_util.py

@@ -127,10 +127,9 @@ def command_owns_foreground_runner(command, *, data_dir: str | Path) -> bool:
 
 def runtime_stack_component_label(*, owner=None, data_dir: str | Path) -> str:
     try:
-        from archivebox.config.common import get_config
         from archivebox.workers.supervisord_util import active_supervisord_runtime_components
 
-        components = active_supervisord_runtime_components(config=get_config())
+        components = active_supervisord_runtime_components()
     except Exception:
         components = []
 

archivebox/core/templatetags/core_tags.py

@@ -1,34 +1,33 @@
+import os
+from pathlib import Path
 from typing import Any
 
+from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
 from django import template
 from django.contrib.admin.templatetags.base import InclusionAdminNode
-from django.utils.safestring import mark_safe
+from django.templatetags.static import static
+from django.utils import timezone
 from django.utils.html import escape
+from django.utils.safestring import mark_safe
+from django.utils.text import Truncator
 
-from pathlib import Path
-
-from abx_plugins.plugins.archivewebpage.replay_preview import is_replay_target as is_archivewebpage_replay_target
-
+from archivebox.core.routes_util import (
+    build_snapshot_url,
+    get_admin_base_url,
+    get_snapshot_base_url,
+    get_web_base_url,
+)
+from archivebox.core.setup_wizard import get_base_url_mismatch_context, get_setup_wizard_context
 from archivebox.plugins.discovery import (
     get_plugin_icon,
-    get_plugin_template,
     get_plugin_name,
+    get_plugin_template,
 )
-from archivebox.core.routes_util import (
-    canonical_base_host_for_request,
-    get_admin_base_url,
-    get_public_base_url,
-    get_web_base_url,
-    get_snapshot_base_url,
-    build_snapshot_url,
-)
-
 
 register = template.Library()
 
 _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml", ".yaml", ".md", ".log")
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
-_MHTML_PREVIEW_EXTS = (".mhtml", ".mht")
 
 _MEDIA_FILE_EXTS = {
     ".mp4",
@@ -92,15 +91,15 @@ def _coerce_output_file_size(value: Any) -> int | None:
 def _count_media_files(result) -> int:
     try:
         output_files = _normalize_output_files(result.output_files or {})
-    except Exception:
+    except (AttributeError, TypeError, ValueError):
         output_files = {}
 
     if output_files:
-        return sum(1 for path in output_files.keys() if Path(path).suffix.lower() in _MEDIA_FILE_EXTS)
+        return sum(1 for path in output_files if Path(path).suffix.lower() in _MEDIA_FILE_EXTS)
 
     try:
         plugin_dir = Path(result.snapshot_dir) / result.plugin
-    except Exception:
+    except (AttributeError, TypeError, ValueError):
         return 0
 
     if not plugin_dir.exists():
@@ -124,7 +123,7 @@ def _list_media_files(result) -> list[dict]:
     media_files: list[dict] = []
     try:
         plugin_dir = Path(result.snapshot_dir) / result.plugin
-    except Exception:
+    except (AttributeError, TypeError, ValueError):
         return media_files
 
     output_files = _normalize_output_files(result.output_files or {})
@@ -182,7 +181,7 @@ def _resolve_snapshot_output_file(snapshot_dir: str | Path | None, raw_output_pa
         snap_dir = Path(snapshot_dir).resolve()
         if snap_dir not in output_file.parents and output_file != snap_dir:
             return None
-    except Exception:
+    except (OSError, RuntimeError, TypeError, ValueError):
         return None
 
     if output_file.exists() and output_file.is_file():
@@ -207,17 +206,17 @@ def _build_snapshot_files_url(snapshot_id: str, request=None, config=None) -> st
     return build_snapshot_url(str(snapshot_id), "/?files=1", request=request, config=config)
 
 
-def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None) -> str:
+def _build_snapshot_preview_url(snapshot_id: str, path: str = "", request=None, config=None, plugin: str = "") -> str:
     if path == "about:blank":
         return path
     if _is_root_snapshot_output_path(path):
         return _build_snapshot_files_url(snapshot_id, request=request, config=config)
     url = build_snapshot_url(str(snapshot_id), path, request=request, config=config)
+    path_parts = Path(path).parts
+    plugin = get_plugin_name(plugin) if plugin else (path_parts[0] if len(path_parts) > 1 else "")
+    has_plugin_preview = bool(plugin and get_plugin_template(plugin, "full", fallback=False))
     if not (
-        _is_text_preview_path(path)
-        or _is_image_preview_path(path)
-        or (path or "").lower().endswith(_MHTML_PREVIEW_EXTS)
-        or is_archivewebpage_replay_target(path or "")
+        _is_text_preview_path(path) or _is_image_preview_path(path) or has_plugin_preview or is_archivewebpage_replay_target(path or "")
     ):
         return url
     separator = "&" if "?" in url else "?"
@@ -266,7 +265,7 @@ def _render_text_file_preview(snapshot_dir: str | Path | None, raw_output_path:
         lines = text.splitlines()[:6]
         snippet = "\n".join(lines)
         return _render_text_preview(plugin, icon_html, snippet)
-    except Exception:
+    except (OSError, UnicodeDecodeError, ValueError):
         return None
 
 
@@ -279,12 +278,12 @@ def split(value, separator: str = ","):
 def index(value, position):
     try:
         return value[int(position)]
-    except Exception:
+    except (IndexError, TypeError, ValueError):
         return None
 
 
 @register.filter
-def file_size(num_bytes: int | float) -> str:
+def file_size(num_bytes: float) -> str:
     for count in ["Bytes", "KB", "MB", "GB"]:
         if num_bytes > -1024.0 and num_bytes < 1024.0:
             return f"{num_bytes:3.1f} {count}"
@@ -357,7 +356,7 @@ def _machine_health_stats() -> dict:
         from archivebox.machine.detect import get_host_stats
 
         stats = get_host_stats() or {}
-    except Exception:
+    except (ImportError, RuntimeError, TypeError, ValueError):
         stats = {}
 
     _health_cache["checked_at"] = now
@@ -373,13 +372,15 @@ def system_warnings_banner(context):
     1. ``mode="unconfigured"``— ``BASE_URL`` is empty. Security/correctness
        issue: until it's pinned, generated URLs can echo any Host the client
        sends, and admin/web/api routing has no canonical anchor.
-    2. ``mode="unsafe"``      — ``SERVER_SECURITY_MODE`` is a non-subdomain
-       mode. Archived pages share an origin with privileged routes.
-    3. ``mode="low_disk"``    — ``DATA_DIR`` has <1 GiB free; new archive
+    2. ``mode="base_url_mismatch"`` — the browser reached ArchiveBox through
+       an origin that differs from the configured canonical ``BASE_URL``.
+    3. ``mode="unsafe"``      — ``SERVER_SECURITY_MODE`` explicitly enables
+       a lower-security replay mode.
+    4. ``mode="low_disk"``    — ``DATA_DIR`` has <1 GiB free; new archive
        jobs will start failing on ENOSPC.
-    4. ``mode="high_memory"`` — virtual memory utilization at/above 95%; the
+    5. ``mode="high_memory"`` — virtual memory utilization at/above 95%; the
        host is one OOM-kill from a crash.
-    5. ``mode="high_load"``   — 15-minute load average exceeds 3 × CPU count
+    6. ``mode="high_load"``   — 15-minute load average exceeds 3 × CPU count
        (the kernel's own sustained-load EMA, so no rolling buffer of ours is
        needed).
 
@@ -395,8 +396,11 @@ def system_warnings_banner(context):
         config = get_config(resolve_plugins=False)
 
     if not config.BASE_URL:
-        return _unconfigured_banner_context(context.get("request"))
-    if not config.USES_SUBDOMAIN_ROUTING:
+        return get_setup_wizard_context(context.get("request"), config)
+    mismatch = get_base_url_mismatch_context(context.get("request"), config)
+    if mismatch:
+        return mismatch
+    if config.IS_LOWER_SECURITY_MODE:
         return {"mode": "unsafe"}
 
     stats = _machine_health_stats()
@@ -415,6 +419,8 @@ def system_warnings_banner(context):
     # "sustained for 15min" and the kernel already maintains that EMA.
     load_15 = cpu_load[2] if isinstance(cpu_load, (list, tuple)) and len(cpu_load) >= 3 else None
     if isinstance(load_15, (int, float)) and load_15 > _HIGH_LOAD_MULTIPLE * cpu_count:
+        if os.environ.get("UI_SCREENSHOT_HIDE_HIGH_LOAD_WARNING") == "1":
+            return {"mode": ""}
         return {
             "mode": "high_load",
             "load_15": f"{load_15:.2f}",
@@ -425,49 +431,6 @@ def system_warnings_banner(context):
     return {"mode": ""}
 
 
-def _unconfigured_banner_context(request) -> dict:
-    """Build the banner payload for the unset-BASE_URL case.
-
-    Always returns ``mode="unconfigured"`` — the user explicitly asked for
-    the banner to render whenever ``BASE_URL`` is empty, regardless of
-    whether the request host happens to match a CSRF-derived value. The
-    ``suggested_base_url`` is derived from the current request when one is
-    available so the user can copy/paste the right value straight into
-    their config.
-    """
-    if request is None:
-        return {
-            "mode": "unconfigured",
-            "actual_host": "",
-            "suggested_base_url": "",
-            "machine_admin_url": "",
-        }
-    scheme = request.scheme or "http"
-    actual_full_host = request.get_host() or ""
-    canonical_host = canonical_base_host_for_request(actual_full_host)
-    # Suggest the wildcard form ``http://*.<host>`` so the value lands in the
-    # operator's clipboard already aligned with subdomain routing. The config
-    # parser strips the leading ``*.`` so users can paste it verbatim.
-    suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
-    user = request.user
-    is_superuser = bool(user and user.is_authenticated and user.is_superuser)
-    machine_admin_url = ""
-    if is_superuser:
-        try:
-            from archivebox.machine.models import Machine
-
-            machine = Machine.current()
-            machine_admin_url = f"/admin/machine/machine/{machine.id}/change/"
-        except Exception:
-            machine_admin_url = ""
-    return {
-        "mode": "unconfigured",
-        "actual_host": actual_full_host,
-        "suggested_base_url": suggested_base_url,
-        "machine_admin_url": machine_admin_url,
-    }
-
-
 @register.simple_tag(takes_context=True)
 def url_replace(context, **kwargs):
     dict_ = context["request"].GET.copy()
@@ -495,11 +458,6 @@ def web_base_url(context) -> str:
     return get_web_base_url(request=context.get("request"), config=context.get("CONFIG"))
 
 
-@register.simple_tag(takes_context=True)
-def public_base_url(context) -> str:
-    return get_public_base_url(request=context.get("request"), config=context.get("CONFIG"))
-
-
 @register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
     snapshot_id = _snapshot_id(snapshot)
@@ -513,9 +471,213 @@ def snapshot_url(context, snapshot, path: str = "") -> str:
 
 
 @register.simple_tag(takes_context=True)
-def snapshot_preview_url(context, snapshot, path: str = "") -> str:
+def snapshot_archiveresult_url(context, snapshot, plugin: str, filename: str) -> str:
+    snapshot_id = str(_snapshot_id(snapshot))
+    url_cache = snapshot.__dict__.setdefault("_snapshot_archiveresult_url_cache", {})
+    cache_key = (plugin, filename)
+    if cache_key in url_cache:
+        return url_cache[cache_key]
+
+    results = None
+    if "_admin_archiveresults" in snapshot.__dict__:
+        results = snapshot.__dict__["_admin_archiveresults"]
+    elif "archiveresult_set" in snapshot.__dict__.get("_prefetched_objects_cache", {}):
+        results = snapshot.archiveresult_set.all()
+    elif "_snapshot_archiveresult_url_results" in snapshot.__dict__:
+        results = snapshot.__dict__["_snapshot_archiveresult_url_results"]
+
+    if results is None:
+        from archivebox.core.models import ArchiveResult
+
+        results = list(
+            ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin__in=("screenshot", "chrome_extension_screenshot", "favicon"),
+                status=ArchiveResult.StatusChoices.SUCCEEDED,
+            ).only(
+                "plugin",
+                "status",
+                "output_files",
+            ),
+        )
+        snapshot.__dict__["_snapshot_archiveresult_url_results"] = results
+
+    for result in results:
+        if result.plugin != plugin or result.status != "succeeded":
+            continue
+        output_files = result.output_files or {}
+        file_info = output_files.get(filename)
+        if not isinstance(file_info, dict) or int(file_info.get("size") or 0) <= 0:
+            continue
+        output_path = filename if file_info.get("root_relative") else f"{plugin}/{filename}"
+        url_cache[cache_key] = build_snapshot_url(snapshot_id, output_path, request=context.get("request"), config=context.get("CONFIG"))
+        return url_cache[cache_key]
+
+    url_cache[cache_key] = ""
+    return ""
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_index_row(context, link) -> str:
+    snapshot_id = str(_snapshot_id(link))
+    request = context.get("request")
+    config = context.get("CONFIG")
+    snapshot_base = get_snapshot_base_url(snapshot_id, request=request, config=config)
+
+    status = getattr(link, "status", None) or "unknown"
+    bookmarked_at = getattr(link, "bookmarked_at", None)
+    timestamp = getattr(link, "timestamp", "")
+    if bookmarked_at:
+        bookmarked_at = timezone.localtime(bookmarked_at)
+        date_text = bookmarked_at.strftime("%Y-%m-%d")
+        time_text = bookmarked_at.strftime("%H:%M")
+        sort_value = str(bookmarked_at.timestamp())
+        title_time = f"Bookmarked: {bookmarked_at} ({timestamp})"
+    else:
+        date_text = ""
+        time_text = ""
+        sort_value = ""
+        title_time = f"Bookmarked: ({timestamp})"
+
+    url = getattr(link, "url", "") or ""
+    title = getattr(link, "title", "") or ""
+    is_pending = status in {"queued", "started", "backoff"}
+    title_text = title or url
+    tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
+    tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
+    if tag_html:
+        tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
+    else:
+        tag_cell = '<span class="empty-value">...</span>'
+
+    num_outputs = int(getattr(link, "num_outputs", 0) or 0)
+    icons = link.icons() if callable(getattr(link, "icons", None)) else getattr(link, "icons", "")
+    icons_cell = str(icons) if icons else '<span class="empty-value">...</span>'
+    archive_size = int(getattr(link, "archive_size", 0) or 0)
+    size_cell = file_size(archive_size) if archive_size else '<span class="empty-value">...</span>'
+    output_plural = "" if num_outputs == 1 else "s"
+
+    if is_pending:
+        preview_html = (
+            '<span class="snapshot-preview snapshot-preview-spinner" aria-label="Archiving in progress">'
+            f'<img src="{escape(static("spinner.gif"))}" alt="" decoding="async" loading="lazy">'
+            "</span>"
+        )
+    elif "_public_preview_paths" in link.__dict__:
+        preview_paths = list(getattr(link, "_public_preview_paths", []) or [])
+        if preview_paths:
+            preview_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in preview_paths]
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+    else:
+        preview_urls = [
+            url
+            for url in (
+                snapshot_archiveresult_url(context, link, "screenshot", "screenshot.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot-1.png"),
+                snapshot_archiveresult_url(context, link, "chrome_extension_screenshot", "screenshot.png"),
+            )
+            if url
+        ]
+        if preview_urls:
+            preview_html = (
+                f'<img src="{escape(preview_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(preview_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" class="snapshot-preview screenshot" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            preview_html = '<span class="snapshot-preview snapshot-preview-empty" aria-label="No preview available"></span>'
+
+    if "_public_favicon_paths" in link.__dict__:
+        favicon_paths = list(getattr(link, "_public_favicon_paths", []) or [])
+        if favicon_paths:
+            favicon_urls = [build_snapshot_url(snapshot_id, path, request=request, config=config) for path in favicon_paths]
+            favicon_html = (
+                f'<img src="{escape(favicon_urls[0])}" '
+                f'data-fallbacks="{escape(",".join(favicon_urls[1:]))}" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+        else:
+            favicon_html = '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
+    else:
+        favicon_url = snapshot_archiveresult_url(context, link, "favicon", "favicon.ico")
+        favicon_html = (
+            (
+                f'<img src="{escape(favicon_url)}" '
+                'data-fallbacks="" '
+                'onerror="nextPublicSnapshotPreview(this)" '
+                'class="link-favicon" alt="" decoding="async" loading="lazy">'
+            )
+            if favicon_url
+            else '<span class="link-favicon link-favicon-empty" aria-hidden="true"></span>'
+        )
+
+    html = f"""
+<tr class="snapshot-row status-{escape(status)}">
+    <td class="snapshot-time" title="{escape(title_time)}" data-sort="{escape(sort_value)}">
+        <a href="{escape(snapshot_base)}/index.html">
+            <span>{escape(date_text)}</span>
+            <small>{escape(time_text)}</small>
+        </a>
+    </td>
+    <td class="snapshot-preview-cell">
+        <a href="{escape(snapshot_base)}/index.html" title="Open archived snapshot">
+            {preview_html}
+        </a>
+    </td>
+    <td class="snapshot-title-cell" title="{escape(title or url)}">
+        <div class="snapshot-title-line">
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-favicon-link" title="Open archived snapshot">
+                {favicon_html}
+            </a>
+            <a href="{escape(snapshot_base)}/index.html" class="snapshot-title">
+                {escape(Truncator(title_text).chars(110))}
+            </a>
+        </div>
+        <a href="{escape(url)}" class="snapshot-url" title="{escape(url)}" target="_blank" rel="noopener noreferrer">
+            {escape(url)}
+        </a>
+        <span class="snapshot-mobile-saved">Saved {escape(date_text)} at {escape(time_text)}</span>
+    </td>
+    <td class="snapshot-tags-cell">
+        {tag_cell}
+    </td>
+    <td class="snapshot-status-cell">
+        <span class="snapshot-status">{escape(status)}</span>
+    </td>
+    <td class="snapshot-files-cell">
+        <span data-number-for="{escape(url)}" title="{num_outputs} successful outputs">
+            {icons_cell}
+        </span>
+    </td>
+    <td class="snapshot-size-cell">
+        <a href="{escape(snapshot_base)}/?files=1" title="View archived files">
+            {size_cell}
+        </a>
+        <small>{num_outputs} output{output_plural}</small>
+    </td>
+</tr>
+"""
+    return mark_safe(html)
+
+
+@register.simple_tag(takes_context=True)
+def snapshot_preview_url(context, snapshot, path: str = "", result=None) -> str:
     snapshot_id = _snapshot_id(snapshot)
-    return _build_snapshot_preview_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
+    plugin = getattr(result, "plugin", "") if result else ""
+    return _build_snapshot_preview_url(
+        str(snapshot_id),
+        path,
+        request=context.get("request"),
+        config=context.get("CONFIG"),
+        plugin=plugin,
+    )
 
 
 @register.simple_tag
@@ -596,8 +758,8 @@ def plugin_card(context, result) -> str:
             # Only return non-empty content (strip whitespace to check)
             if rendered.strip():
                 return mark_safe(rendered)
-    except Exception:
-        pass
+    except (template.TemplateSyntaxError, AttributeError, TypeError, ValueError):
+        rendered = ""
 
     if force_text_preview:
         preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
@@ -672,7 +834,7 @@ def plugin_full(context, result) -> str:
         if rendered.strip():
             return mark_safe(rendered)
         return ""
-    except Exception:
+    except (template.TemplateSyntaxError, AttributeError, TypeError, ValueError):
         return ""
 
 

archivebox/core/urls.py

@@ -27,13 +27,8 @@ from archivebox.core.views import (
     HealthCheckView,
 )
 from archivebox.progressmonitor.views import live_progress_view
-from abx_plugins.plugins.opencode.views import opencode_proxy_view
-
-
-# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
-# from archivebox.config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE
-# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': VERSIONS_AVAILABLE, 'CAN_UPGRADE': CAN_UPGRADE}
-
+from archivebox.search.views import public_snapshot_search_stream_view
+from archivebox.opencode.views import opencode_proxy_view
 
 CONFIG = get_config()
 DEBUG = CONFIG.DEBUG or ("--debug" in sys.argv)
@@ -43,8 +38,9 @@ urlpatterns = [
     path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
     path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
     path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
-    re_path(r"^admin/agent/?(?=$|opencode)", include("abx_plugins.plugins.opencode.urls")),
+    re_path(r"^admin/agent/?(?=$|opencode)", include("archivebox.opencode.urls")),
     re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
+    path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
     path("public/", PublicIndexView.as_view(), name="public-index"),
     path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
     path("archive/", RedirectView.as_view(url="/")),
@@ -84,7 +80,6 @@ urlpatterns = [
     path("api/", include("archivebox.api.urls"), name="api"),
     path("health/", HealthCheckView.as_view(), name="healthcheck"),
     path("error/", lambda request: _raise_test_error(request)),
-    # path('jet_api/', include('jet_django.urls')),  Enable to use https://www.jetadmin.io/integrations/django
     path("index.html", RedirectView.as_view(url="/")),
     path("", HomepageView.as_view(), name="Home"),
 ]
@@ -99,36 +94,3 @@ if getattr(settings, "DEBUG_TOOLBAR", False):
 
 if getattr(settings, "DEBUG_REQUESTS_TRACKER", False) and find_spec("requests_tracker"):
     urlpatterns += [path("__requests_tracker__/", include("requests_tracker.urls"))]
-
-
-# # Proposed FUTURE URLs spec
-# path('',                 HomepageView)
-# path('/add',             AddView)
-# path('/public',          PublicIndexView)
-# path('/snapshot/:slug',  SnapshotView)
-
-# path('/admin',           admin.site.urls)
-# path('/accounts',        django.contrib.auth.urls)
-
-# # Proposed REST API spec
-# # :slugs can be uuid, short_uuid, or any of the unique index_fields
-# path('api/v1/'),
-# path('api/v1/core/'                      [GET])
-# path('api/v1/core/snapshot/',            [GET, POST, PUT]),
-# path('api/v1/core/snapshot/:slug',       [GET, PATCH, DELETE]),
-# path('api/v1/core/archiveresult',        [GET, POST, PUT]),
-# path('api/v1/core/archiveresult/:slug',  [GET, PATCH, DELETE]),
-# path('api/v1/core/tag/',                 [GET, POST, PUT]),
-# path('api/v1/core/tag/:slug',            [GET, PATCH, DELETE]),
-
-# path('api/v1/cli/',                      [GET])
-# path('api/v1/cli/{add,list,config,...}', [POST]),  # pass query as kwargs directly to `run_subcommand` and return stdout, stderr, exitcode
-
-# path('api/v1/extractors/',                    [GET])
-# path('api/v1/extractors/:extractor/',         [GET]),
-# path('api/v1/extractors/:extractor/:func',    [GET, POST]),  # pass query as args directly to chosen function
-
-# future, just an idea:
-# path('api/v1/scheduler/',                [GET])
-# path('api/v1/scheduler/task/',           [GET, POST, PUT]),
-# path('api/v1/scheduler/task/:slug',      [GET, PATCH, DELETE]),