Bump release version to 0.9.72rc28

Resolve relative cookie files before hooks change cwd, reconcile file edits before machine startup saves, and prevent stale cached binaries overriding runtime selections. Cover issue #1896 and real persona localStorage capture through CLI regressions.

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 
 ### Development Setup
 

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -0,0 +1,38 @@
+# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
+# Manual run from the repository root with Lychee 0.24.2:
+# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
+# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
+# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
+include_fragments = "anchor-only"
+include_verbatim = true
+max_retries = 0
+max_concurrency = 4
+timeout = 20
+no_progress = true
+hidden = true
+extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
+exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
+# Includes override this exclude; this is a documentation-link allowlist.
+exclude = ['.*']
+include = [
+ '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
+ '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
+ '^https?://docs\.archivebox\.io(?:/|$)',
+ '^file:.*\.md(?:#|$)',
+ '^file:.*/docs/[^/.#]+(?:#|$)',
+]
+fallback_extensions = ["md"]
+remap = [
+ # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
+
+ '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
+ '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
+ # The wiki accepts this case alias, but raw file paths are case-sensitive.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
+
+ # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
+ # Keep fragment URLs on the rendered wiki to catch published heading drift.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
+]
+preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/artifacts.py

@@ -14,7 +14,7 @@ def trusted(run, repo, workflow, branch, successful=True):
     return (
         (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
         and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch")
+        and run["event"] in ("push", "workflow_dispatch", "schedule")
         and run["path"] == f".github/workflows/{workflow}"
         and (not successful or run["conclusion"] == "success")
     )

.github/workflows/ci.yml

@@ -45,13 +45,6 @@ jobs:
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
-  documentation:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
   codeql:
     needs: prepare
     if: needs.prepare.outputs.run_jobs == 'true'
@@ -83,7 +76,6 @@ jobs:
       - lint
       - install-cli-platform
       - tests
-      - documentation
       - codeql
       - python-artifacts
       - docker-digests
@@ -94,7 +86,6 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
           TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
           DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
@@ -103,7 +94,6 @@ jobs:
             "$LINT_RESULT" \
             "$INSTALL_CLI_PLATFORM_RESULT" \
             "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \
             "$DOCKER_DIGESTS_RESULT"

.github/workflows/docker.yml

@@ -350,6 +350,7 @@ jobs:
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
+        id: mount_dependencies
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         run: |
@@ -371,6 +372,15 @@ jobs:
             test -x "$binary"
           done
 
+      - name: Diagnose failed mount dependency installation
+        if: failure() && steps.mount_dependencies.conclusion == 'failure'
+        shell: bash
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          sudo journalctl --since '-10 minutes' --no-pager \
+            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
       - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash

.github/workflows/docs-links.yml

@@ -0,0 +1,38 @@
+name: Documentation link checks
+
+on:
+  schedule:
+    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHONIOENCODING: utf-8
+
+jobs:
+  links:
+    name: Documentation links and anchors
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+      - name: List tracked documentation and source inputs
+        run: |
+          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
+      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
+        with:
+          lycheeVersion: v0.24.2
+          args: >-
+            --config .github/lychee.toml
+            --root-dir "${{ github.workspace }}"
+            --files-from "${{ runner.temp }}/link-inputs.txt"
+          fail: true
+          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,7 +1,17 @@
 name: Documentation tests
 
 on:
-  workflow_call:
+  schedule:
+    - cron: '7 5,17 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHONIOENCODING: utf-8
@@ -9,7 +19,18 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   discover:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -158,3 +179,17 @@ jobs:
           "$SUDO_BINARY" \
             --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
             "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Diagnose failed Linux package installation
+        if: failure() && runner.os == 'Linux'
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, discover, examples]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/screenshots.yml

@@ -1,38 +1,31 @@
 name: Capture publicsite screenshots
 
 on:
-  push:
-    branches:
-      - dev
-    paths:
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/screenshots.yml
-      - .github/workflows/prepare-release.yml
+  schedule:
+    - cron: '37 5,17 * * *'
   workflow_dispatch:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
 
   deploy:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
@@ -78,3 +71,11 @@ jobs:
           path: publicsite/screenshots
           if-no-files-found: error
           retention-days: 90
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, deploy]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

README.md

@@ -123,13 +123,13 @@ curl -fsSL 'https://get.archivebox.io' | bash
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
-- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Python API](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) for local-only mode)
 - Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
@@ -220,7 +220,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
-<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
+<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-uv-install-shortcut">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
@@ -431,7 +431,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -613,7 +613,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
 <br/>
 </div>
 
@@ -641,7 +641,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
-  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
+  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#import-a-list-of-urls-from-a-text-file).</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
@@ -670,7 +670,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
-See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
+See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) page for documentation and examples.
 
 It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
 
@@ -811,7 +811,7 @@ Installing directly on <strong>Windows without Docker or WSL/WSL2/Cygwin is not
 
 <h4>Learn More</h4>
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Wiki: Install (Dependencies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Wiki: Install (Dependencies)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Wiki: Chromium Install</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Wiki: Upgrading or Merging Archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Wiki: Troubleshooting (Installing)</a></li>
@@ -865,7 +865,7 @@ Each snapshot subfolder includes static metadata and plain extractor output file
 <h4>Learn More</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
@@ -951,7 +951,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -1004,7 +1004,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1065,7 +1065,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <h4>Learn More</h4>
 
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
@@ -1190,7 +1190,7 @@ By having users store their content locally or within their organizations, we ca
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
-ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
+ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
@@ -1199,7 +1199,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
-For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
+For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)...
 
 ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
 
@@ -1254,7 +1254,7 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
-- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox#output-formats)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced

archivebox/api/v1_api.py

@@ -2,6 +2,8 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
+from functools import lru_cache
+from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -10,6 +12,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
+from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -49,6 +52,16 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
+    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
+        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
+        return self._schema_for_prefix(prefix)
+
+    @lru_cache(maxsize=8)
+    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
+        # Routes are registered at startup. Server discovery can reuse their
+        # schema while preserving distinct URL prefixes for mounted instances.
+        return super().get_openapi_schema(path_prefix=path_prefix)
+
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 

archivebox/api/v1_personas.py

@@ -5,9 +5,11 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import UUID
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -108,10 +110,9 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
+        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
         .order_by("created_at")
         .first()
     )
@@ -138,7 +139,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    persona = find_persona(payload.extension_persona_id, name)
+    try:
+        persona = find_persona(payload.extension_persona_id, name)
+        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
+    except ValidationError as err:
+        raise HttpError(409, "; ".join(err.messages)) from err
     created = persona is None
     if persona is None:
         persona = Persona(name=name)

archivebox/base_models/models.py

@@ -282,7 +282,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +301,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/__init__.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
+from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -219,6 +220,12 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
+        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
+        if supervisor_util is not None:
+            # Short commands can start a foreground-owned search daemon too.
+            # Finish its cleanup before another CLI can borrow a dying stack.
+            with redirect_stdout(sys.stderr):
+                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_init.py

@@ -116,6 +116,11 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     # Registration requires the final schema and reconciles legacy file config.
     Machine.current()
 
+    from archivebox.personas.models import Persona
+
+    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
+    Persona.get_or_create_default().ensure_dirs()
+
     from archivebox.misc.db import database_display_location, is_postgres
 
     assert database_exists()

archivebox/cli/archivebox_persona.py

@@ -163,8 +163,9 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(
-            name=name,
+        created = Persona.find_named(name) is None
+        persona = Persona.get_or_create_named(
+            name,
             defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
         )
 
@@ -313,7 +314,9 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.objects.get(name=old_name)
+                persona = Persona.find_named(old_name)
+                if persona is None:
+                    raise Persona.DoesNotExist
 
             # Apply updates from CLI flags
             if name:
@@ -325,7 +328,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = name
+                persona.name = persona.validate_name(name)
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_server.py

@@ -260,6 +260,12 @@ def server(
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
+    from archivebox.personas.models import Persona
+
+    # Older collections deferred this until /add/. Reconcile before serving any
+    # requests, and report ambiguous profile names before Chrome can use them.
+    Persona.get_or_create_default().ensure_dirs()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         from archivebox.core.routes_util import build_admin_url
 

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -468,6 +487,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -498,7 +522,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -825,6 +849,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +875,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")
@@ -1156,11 +1189,15 @@ def get_config(
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+        machine_config = machine.config
+        if resolve_plugins:
+            from archivebox.machine.models import _sanitize_machine_config
+
+            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                machine_config,
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_snapshots.py

@@ -52,6 +52,7 @@ from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
 from archivebox.progressmonitor.views import progress_endpoint
+from archivebox.workers.models import RETRY_AT_MAX
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
@@ -131,11 +132,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +993,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1540,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.update(status=Snapshot.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/forms.py

@@ -304,13 +304,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
-        default_persona = Persona.get_or_create_default()
+        # Persona creation belongs to init/server startup, never a GET of /add/.
         persona_queryset = Persona.objects.order_by("name")
         if not self.can_override_crawl_config:
             persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         self.fields["persona"].queryset = persona_queryset
 
-        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        selected_persona = persona_queryset.filter(name__iexact="Default").first() or persona_queryset.first()
         default_config = get_config(persona=selected_persona) if selected_persona else get_config()
         if selected_persona:
             self.fields["persona"].initial = selected_persona.name
@@ -338,6 +338,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
                 get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
+    def clean_persona(self):
+        persona = self.cleaned_data.get("persona")
+        if persona:
+            persona.validate_name(persona.name)
+        return persona
+
     def clean(self):
         cleaned_data = super().clean() or {}
 

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -175,6 +176,19 @@ class SnapshotTag(models.Model):
 class SnapshotQuerySet(models.QuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -541,6 +555,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -574,7 +589,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -661,6 +676,49 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).update(status=self.DELETING_STATE, retry_at=RETRY_AT_MAX)
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        # Finish the bounded read before touching files: even an open SELECT
+        # cursor can hold a SQLite read lock for the duration of slow cleanup.
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""
@@ -1198,6 +1256,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         target_dir = self.get_storage_path_for_version(target)
         if target_dir.exists():
+            self.reconcile_with_index(output_dir=target_dir, update_existing_archive_results=False)
             self.hydrate_archiveresult_output_metadata(snapshot_dir=target_dir)
         if cleanup:
             old_dir, new_dir = cleanup

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/machine/models.py

@@ -255,40 +255,40 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Reconcile once, on the first Machine.current() call in this process.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
         # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine

archivebox/misc/checks.py

@@ -194,7 +194,10 @@ def check_not_root():
     if os.geteuid() == 0 and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
         print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
-        print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
+        print(
+            "        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user",
+            file=sys.stderr,
+        )
 
 
 def is_archivebox_source_root(path: Path | str | None = None) -> bool:
@@ -255,7 +258,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",

archivebox/misc/shell_welcome_message.py

@@ -48,10 +48,10 @@ if __name__ == "__main__":
 
     prnt("[i] :heavy_dollar_sign: Welcome to the ArchiveBox Shell!")
     prnt(
-        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage[/link]",
+        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage[/link]",
     )
     prnt(
-        "          [link=https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html[/link]",
+        "          [link=https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html[/link]",
     )
     prnt()
     prnt(" :grey_question: [violet]Hint[/] [i]Here are some examples to get started:[/]")

archivebox/personas/forms.py

@@ -122,14 +122,14 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
                 "Use the custom path/CDP option if the browser data lives elsewhere."
             )
 
-        self.build_plugin_groups(get_config(persona=self.instance) if self.instance and self.instance.pk else get_config())
+        self.build_plugin_groups(get_config(persona=self.instance) if not self.instance._state.adding else get_config())
 
     def clean_name(self) -> str:
         name = str(self.cleaned_data.get("name") or "").strip()
         is_valid, error_message = validate_persona_name(name)
         if not is_valid:
             raise forms.ValidationError(error_message)
-        return name
+        return self.instance.validate_name(name)
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()

archivebox/personas/models.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
 from django.db import IntegrityError, models
+from django.core.exceptions import ValidationError
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -74,8 +75,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -117,6 +122,13 @@ class Persona(ModelWithConfig):
         app_label = "personas"
 
     def save(self, *args, **kwargs):
+        # Adopt the spelling on disk, never rename a Chrome profile just to fix case.
+        self.__dict__.pop("_persona_path", None)
+        canonical_name = self.validate_name(self.name)
+        if canonical_name != self.name:
+            self.name = canonical_name
+            if kwargs.get("update_fields") is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*kwargs["update_fields"], "name"]))
         config = dict(self.config or {})
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
@@ -133,10 +145,59 @@ class Persona(ModelWithConfig):
 
     @property
     def path(self) -> Path:
-        """Path to persona directory under PERSONAS_DIR."""
+        """Reuse the directory's spelling without renaming a Chrome profile."""
         from archivebox.config.constants import CONSTANTS
 
-        return CONSTANTS.PERSONAS_DIR / self.name
+        cached = self.__dict__.get("_persona_path")
+        if cached is None or cached[0] != self.name:
+            directory_name = self.directory_name(self.name)
+            cached = (self.name, CONSTANTS.PERSONAS_DIR / directory_name)
+            self.__dict__["_persona_path"] = cached
+        return cached[1]
+
+    @staticmethod
+    def directory_name(name: str) -> str:
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.personas.importers import validate_persona_name
+
+        valid, error = validate_persona_name(name)
+        if not valid:
+            raise ValidationError(error)
+        # Linux can hold both Default/default; macOS usually aliases them. Reuse a
+        # single match on either filesystem, but never guess which conflicting
+        # profile contains the user's login state. Only list names, not profile trees.
+        try:
+            with os.scandir(CONSTANTS.PERSONAS_DIR) as entries:
+                matches = sorted(entry.name for entry in entries if entry.name.casefold() == name.casefold())
+        except FileNotFoundError:
+            matches = []
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona directories: {'; '.join(str(CONSTANTS.PERSONAS_DIR / match) for match in matches)}. "
+                "Stop Chrome and rename the conflicting directories manually to distinct names; no profile directories were renamed.",
+            )
+        return matches[0] if matches else name
+
+    @classmethod
+    def find_named(cls, name: str) -> "Persona | None":
+        # SQLite's iexact is ASCII-only; Python casefold gives every entry point
+        # the same Unicode matching rules as the filesystem-name lookup above.
+        matches = [(pk, existing) for pk, existing in cls.objects.values_list("pk", "name") if existing.casefold() == name.casefold()]
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona records: {'; '.join(f'{existing!r} (ID {pk})' for pk, existing in matches)}. "
+                "Rename the conflicting personas manually to distinct names; profile files have not been changed.",
+            )
+        return cls.objects.get(pk=matches[0][0]) if matches else None
+
+    def validate_name(self, name: str) -> str:
+        existing = self.find_named(name)
+        if existing is not None and existing.pk != self.pk:
+            raise ValidationError(
+                f"Name conflicts with persona {existing.name!r} (ID {existing.pk}, directory {existing.path}). "
+                "Choose a distinct name or rename that persona record manually.",
+            )
+        return self.directory_name(name)
 
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
@@ -156,9 +217,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -171,10 +235,13 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
+        return derive_persona_config(name=self.path.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
+        # Recheck at profile preparation, including changes made outside this process.
+        self.__dict__.pop("_persona_path", None)
+        self.validate_name(self.name)
         self.path.mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
@@ -246,16 +313,26 @@ class Persona(ModelWithConfig):
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
     @classmethod
-    def get_or_create_named(cls, name: str) -> "Persona":
+    def get_or_create_named(cls, name: str, *, defaults: dict | None = None) -> "Persona":
+        # Init/startup must reuse a user-created 'default', not create a second
+        # DB row pointing at a different (or shared) profile (issue #1896).
         persona_name = (name or "Default").strip() or "Default"
-        persona = cls.objects.filter(name=persona_name).first()
+        persona = cls.find_named(persona_name)
         if persona is not None:
+            canonical_name = persona.validate_name(persona.name)
+            if canonical_name != persona.name:
+                persona.name = canonical_name
+                persona.save(update_fields=["name"])
             return persona
 
         try:
-            return cls.objects.create(name=persona_name)
-        except IntegrityError:
-            return cls.objects.get(name=persona_name)
+            return cls.objects.create(name=persona_name, **(defaults or {}))
+        except (IntegrityError, ValidationError):
+            persona = cls.find_named(persona_name)
+            if persona is None:
+                raise
+            persona.validate_name(persona.name)
+            return persona
 
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
@@ -315,20 +392,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +424,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/services/runner.py

@@ -65,6 +65,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -738,7 +739,14 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(machine.config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
         if self.persona:
             self.base_config.update(
@@ -1245,6 +1253,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1260,6 +1269,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1283,6 +1293,7 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
                 if self.execution_interrupted():
                     return
                 if snapshot["status"] == "sealed":
@@ -1306,6 +1317,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1981,14 +2002,35 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read

archivebox/templates/core/setup_wizard.html

@@ -35,7 +35,7 @@
       <div class="abx-question-options" id="archivebox-setup-dns-options">
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="localhost"> <span><b>⭐ Localhost</b><small>No setup needed. <code id="archivebox-setup-localhost-dns-example">*.archivebox.localhost</code> works directly in modern browsers with all features.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="single"> <span><b>⚠️ Single-domain DNS</b><small>Register a DNS record like <code>A archivebox.example.com</code> → <code>your server's IP</code>, or add an equivalent <code>/etc/hosts</code> entry pointing to your server. Some archived content that depends on JavaScript may not display properly in this mode. JavaScript still runs during capture, but viewing HTML captured by <code>wget</code>, <code>dom</code>, <code>responses</code>, <code>staticfile</code>, and similar plugins will not replay JavaScript unless wildcard DNS is used.</small></span></label>
-        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
+        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
       </div>
       <div class="abx-question-status" id="archivebox-setup-dns-status">Choose a DNS mode to begin testing.</div>
     </fieldset>

archivebox/templates/core/system_warnings_banner.html

@@ -27,7 +27,7 @@
       Only <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ free_gb }} GiB</code>
       free on <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">DATA_DIR</code>
       — new archive jobs will fail until space is reclaimed.
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Disk-Usage" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#purging-entries" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
     {% elif mode == "high_memory" %}
       Virtual memory at <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ mem_pct }}%</code>
       — the host is close to OOM. Consider stopping crawls or scaling up.
@@ -35,7 +35,7 @@
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
     {% elif mode == "base_url_mismatch" %}
       Editing disabled from unrecognized URL. Visit
       <a href="{{ configured_base_url }}" style="color:#fff;text-decoration:underline;text-transform:none;">{{ configured_base_url }}</a>

archivebox/tests/test_api_discovery_performance.py

@@ -0,0 +1,54 @@
+"""Server discovery must not repeat schema generation or probe installed tools."""
+
+import sys
+import threading
+import os
+from pathlib import Path
+
+import pytest
+from django.test.utils import override_script_prefix
+
+from archivebox.config import VERSION
+from archivebox.machine.models import Machine
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("script_name", ["", "/mounted"])
+def test_repeated_discovery_reuses_schema_without_binary_filesystem_probes(client, hermetic_lib_dir, script_name):
+    install_real_binary("node", binproviders="env,apt,brew")
+    resolve_abxpkg_binary_env(hermetic_lib_dir, "node")
+    machine = Machine.current()
+    machine.config = {**machine.config, "NODE_BINARY": str(hermetic_lib_dir / "env" / "bin" / "node")}
+    machine.save(update_fields=["config"])
+    request = {"HTTP_HOST": "api.archivebox.localhost:5797", "SCRIPT_NAME": script_name}
+    with override_script_prefix(script_name):
+        first = client.get("/api/v1/openapi.json", **request)
+    assert first.status_code == 200, first.content
+    schema_calls = []
+    filesystem_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name == "model_json_schema":
+            schema_calls.append(frame.f_code.co_filename)
+        if (event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}) or (
+            event == "c_call" and arg in (os.stat, os.lstat)
+        ):
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(hermetic_lib_dir):
+                filesystem_calls.append(str(path))
+
+    previous = sys.getprofile()
+    previous_thread = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        with override_script_prefix(script_name):
+            second = client.get("/api/v1/openapi.json", **request)
+    finally:
+        threading.setprofile_all_threads(previous_thread)
+        sys.setprofile(previous)
+    assert second.status_code == 200, second.content
+    assert second.json() == first.json()
+    assert second.json()["info"]["version"] == VERSION
+    assert f"{script_name}/api/v1/core/snapshots" in second.json()["paths"]
+    assert (len(schema_calls), filesystem_calls) == (0, [])

archivebox/tests/test_cli_background_screenshot.py

@@ -0,0 +1,39 @@
+"""Exercise the browser target lifecycle through a real multi-page CLI crawl."""
+
+import os
+import sqlite3
+import subprocess
+
+import pytest
+
+
+@pytest.mark.timeout(180)
+def test_cli_crawl_captures_background_screenshots(tmp_path):
+    env = {**os.environ, "SCREENSHOT_TIMEOUT": "5"}
+    initialized = subprocess.run(
+        ["archivebox", "init", "--quick"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    assert initialized.returncode == 0, initialized.stderr
+    captured = subprocess.run(
+        ["archivebox", "add", "--plugins=title,wget,screenshot", "https://example.com", "https://example.org"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert captured.returncode == 0, captured.stderr
+    with sqlite3.connect(tmp_path / "index.sqlite3") as db:
+        results = db.execute("SELECT status, output_str FROM core_archiveresult WHERE plugin='screenshot'").fetchall()
+    assert len(results) == 2, captured.stdout
+    if any(status != "succeeded" for status, _ in results):
+        print(captured.stdout, captured.stderr)
+    assert all(status == "succeeded" for status, _ in results), (results, captured.stdout, captured.stderr)
+    images = list((tmp_path / "archive" / "users").glob("*/snapshots/**/screenshot/screenshot.png"))
+    assert len(images) == 2
+    assert all(image.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") and image.stat().st_size > 1000 for image in images)

archivebox/tests/test_cli_persona.py

@@ -50,12 +50,14 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
     """Import an actual Chromium-created session and archive its private page."""
     import hashlib
     import json
+    import shlex
     import subprocess
 
     from werkzeug.wrappers import Response
     from archivebox.tests.conftest import resolve_abxpkg_chrome_env
 
     browser_env = resolve_abxpkg_chrome_env(tmp_path / "lib")
+    browser_env["ABXPKG_LIB_DIR"] = str(tmp_path / "lib")
     source = tmp_path / "browser"
     httpserver.expect_request("/login").respond_with_data(
         '<html><title>Signed in</title><script>localStorage.setItem("persona-preference", "forest-green")</script>Signed in</html>',
@@ -90,6 +92,7 @@ const puppeteer = require(process.argv[1]);
         const page = await browser.newPage();
         await page.goto(process.argv[4]);
         console.log(await page.title());
+        console.log(await page.evaluate(() => localStorage.getItem('persona-preference')));
     } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exit(1); });
 """,
@@ -104,6 +107,7 @@ const puppeteer = require(process.argv[1]);
     )
     assert seeded.returncode == 0, seeded.stderr
     assert "Signed in" in seeded.stdout
+    assert "forest-green" in seeded.stdout
     preferences = source / "Profile 2" / "Preferences"
     original_preferences = preferences.read_bytes()
 
@@ -161,6 +165,9 @@ const puppeteer = require(process.argv[1]);
         env={**browser_env, "CHROME_SANDBOX": "false"},
     )
     assert archived.returncode == 0, archived.stderr
+    launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
+    assert len(launch_commands) == 1
+    assert shlex.split(launch_commands[0].read_text().splitlines()[1])[0] == browser_env["CHROME_BINARY"]
     outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
     assert len(outputs) == 1
     html = outputs[0].read_text()

archivebox/tests/test_cli_server.py

@@ -448,7 +448,10 @@ def test_port_compatibility_does_not_add_listeners_outside_docker_defaults(tmp_p
         check=True,
     )
     workers = json.loads(result.stdout)
-    assert [worker["name"] for worker in workers] == ["worker_runserver" if debug else "worker_daphne", "worker_runner"]
+    assert [worker["name"] for worker in workers] == [
+        "worker_runserver" if debug else "worker_daphne",
+        "worker_runner",
+    ]
     command = shlex.split(workers[0]["command"])
     assert "--endpoint" not in command
     assert (f"127.0.0.1:{port}" if debug else f"--port={port}") in command

archivebox/tests/test_cli_update.py

@@ -19,6 +19,50 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_rescan_recovers_already_migrated_idless_result(initialized_archive, tmp_path):
+    """Repair collections already migrated by versions affected by #1899."""
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(["add", "--index-only", "https://example.com/legacy-result"], env=env, check=True)
+    run_queued_crawls(initialized_archive, env)
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url="https://example.com/legacy-result")
+        snapshot.write_index_jsonl()
+        output_dir = snapshot.output_dir
+        snapshot_id = snapshot.pk
+    payload = b"<html>Recovered after a previous migration</html>"
+    (output_dir / "singlefile.html").write_bytes(payload)
+    with (output_dir / "index.jsonl").open("a") as index:
+        index.write(
+            json.dumps(
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(snapshot_id),
+                    "plugin": "singlefile",
+                    "hook_name": "",
+                    "status": "succeeded",
+                    "output_str": "singlefile.html",
+                    "start_ts": "2021-03-07T23:56:16-08:00",
+                },
+            )
+            + "\n",
+        )
+    result_id = None
+    for expected in ("Repaired: 1", "Unchanged: 1"):
+        result = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert result.returncode == 0, result.stdout + result.stderr
+        assert expected in result.stdout
+        with use_archivebox_db(initialized_archive):
+            archived = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="singlefile")
+            assert archived.status == "succeeded"
+            assert archived.output_size == len(payload)
+            assert archived.output_files
+            assert archived.snapshot.output_size == len(payload)
+            if result_id is not None:
+                assert archived.pk == result_id
+            result_id = archived.pk
+        assert (output_dir / "singlefile.html").read_bytes() == payload
+
+
 def test_rescan_imports_current_layout_and_is_idempotent(initialized_archive, tmp_path):
     env = cli_env(disable_extractors=True)
     run_archivebox_cmd(["add", "--index-only", "https://example.com/rescan"], cwd=initialized_archive, env=env, check=True)

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -15,6 +15,49 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.parametrize("index_format", ["history", "archive_results"])
+def test_rescan_imports_legacy_result_rows_and_saved_content(tmp_path, initialized_archive, index_format):
+    """Issue #1899: orphan metadata must become visible results on the first import."""
+    env = cli_env(disable_extractors=True)
+    legacy_dir = tmp_path / "archive" / "1615190176"
+    legacy_dir.mkdir(parents=True)
+    payload = b"<html><title>Recovered capture</title><body>saved content</body></html>"
+    (legacy_dir / "singlefile.html").write_bytes(payload)
+    entry = {
+        "plugin": "singlefile",
+        "status": "succeeded",
+        "output": "singlefile.html",
+        "start_ts": "2021-03-07T23:56:16-08:00",
+        "end_ts": "2021-03-07T23:56:16-08:00",
+    }
+    metadata = {
+        "url": "https://example.com/recovered",
+        "timestamp": legacy_dir.name,
+        "title": "Recovered capture",
+        index_format: {"singlefile": [entry]} if index_format == "history" else [entry],
+    }
+    original_json = json.dumps(metadata)
+    (legacy_dir / "index.json").write_text(original_json)
+    result_id = None
+    for _ in range(2):
+        command = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert command.returncode == 0, command.stdout + command.stderr
+        with use_archivebox_db(tmp_path):
+            snapshot = Snapshot.objects.get(url=metadata["url"])
+            result = snapshot.archiveresult_set.get(plugin="singlefile")
+            assert result.status == "succeeded"
+            assert result.output_size == len(payload)
+            assert result.output_files
+            assert snapshot.status == "sealed"
+            assert snapshot.archiveresult_set.count() == 1
+            assert snapshot.output_size == len(payload)
+            assert (snapshot.output_dir / "singlefile.html").read_bytes() == payload
+            assert (snapshot.output_dir / "index.json").read_text() == original_json
+            if result_id is not None:
+                assert result.pk == result_id
+            result_id = result.pk
+
+
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)

archivebox/tests/test_config_DELETE_AFTER.py

@@ -160,6 +160,10 @@ print(json.dumps({{
 
 
 def test_delete_after_real_add_page_and_rest_create_paths(client):
+    from archivebox.personas.models import Persona
+
+    # Match init/server startup; visiting /add/ must not create persona records.
+    Persona.get_or_create_default()
     User = get_user_model()
     admin_user = User.objects.create_superuser(
         username="retentionadmin",

archivebox/tests/test_core_config.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 from pydantic import ValidationError
@@ -95,3 +96,40 @@ def test_resolving_scoped_config_does_not_mutate_process_environment(tmp_path):
             os.environ.pop("ABXPKG_LIB_DIR", None)
         else:
             os.environ["ABXPKG_LIB_DIR"] = previous_lib_dir
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE", "WGET_COOKIES_FILE", "YTDLP_COOKIES_FILE", "GALLERYDL_COOKIES_FILE"])
+def test_cookie_paths_remain_collection_relative_after_runtime_overlays(key, tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: "auth/cookies.txt"})
+    assert str(config[key]) == str(CONSTANTS.DATA_DIR / "auth/cookies.txt")
+    runtime = config.for_crawl_runtime(runtime_overrides={key: "other/cookies.txt"})
+    assert runtime[key] == str(CONSTANTS.DATA_DIR / "other/cookies.txt")
+    absolute = tmp_path / "absolute-cookies.txt"
+    config = ArchiveBoxConfig(**{key: str(absolute)})
+    assert str(config[key]) == str(absolute)
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE"])
+@pytest.mark.parametrize("value", ["", "  ", None])
+def test_empty_cookie_path_disables_import(key, value):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: value})
+    assert config[key] is None
+    assert key not in config.for_crawl_runtime()
+
+
+def test_cookie_runtime_path_preserves_configured_symlink(tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    export = tmp_path / "export.txt"
+    export.write_text("# Netscape HTTP Cookie File\n")
+    configured = tmp_path / "cookies.txt"
+    configured.symlink_to(export)
+    config = ArchiveBoxConfig(COOKIES_FILE=configured)
+    assert str(config.COOKIES_FILE) == str(configured)
+    runtime = config.for_crawl_runtime()
+    assert runtime["COOKIES_FILE"] == str(configured)
+    assert Path(runtime["COOKIES_FILE"]).read_bytes() == export.read_bytes()

archivebox/tests/test_crawl_history.py

@@ -0,0 +1,111 @@
+"""Completed captures belong in the database, not a crawl-long event history."""
+
+import asyncio
+import gc
+import json
+import threading
+import weakref
+from collections import Counter
+
+import pytest
+
+from abx_dl.events import ProcessCompletedEvent, SnapshotCompletedEvent, SnapshotEvent
+from werkzeug import Response
+from archivebox.base_models.models import get_or_create_system_user_pk
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.services.runner import CrawlRunner, run_install
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("timeout_page", [False, True])
+def test_completed_snapshots_expire_after_database_projection(tmp_path, httpserver, hermetic_lib_dir, timeout_page):
+    urls = []
+    response_started = threading.Event()
+    release_response = threading.Event()
+
+    def stalled_response(_request):
+        def body():
+            response_started.set()
+            yield b"<html><head><title>Saved page 3</title></head><body>"
+            release_response.wait()
+            yield b"Completed response after the browser timed out.</body></html>"
+
+        return Response(body(), content_type="text/html")
+
+    httpserver.expect_request("/favicon.ico").respond_with_data("", status=204)
+    for index in range(8):
+        path = f"/page/{index}"
+        if timeout_page and index == 3:
+            httpserver.expect_request(path).respond_with_handler(stalled_response)
+            urls.append(httpserver.url_for(path))
+            continue
+        httpserver.expect_request(path).respond_with_data(
+            f"<html><head><title>Saved page {index}</title></head><body>"
+            "This real page is part of a multi-snapshot crawl whose completed captures "
+            "must remain available in the database after their events expire.</body></html>",
+            content_type="text/html",
+        )
+        urls.append(httpserver.url_for(path))
+    run_install(plugin_names=["title"])
+    crawl = Crawl.objects.create(
+        urls="\n".join(urls),
+        config={
+            "ABXPKG_LIB_DIR": str(hermetic_lib_dir),
+            "PLUGINS": "title",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": 1,
+            "CHROME_PAGELOAD_TIMEOUT": 5,
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
+    runner.bus.event_history.max_history_size = 120
+    sizes = []
+    references = []
+    counts = Counter()
+    retired_ids = set()
+    retained_completed = []
+
+    def count_event(event):
+        counts[event.event_type] += 1
+
+    def observe_snapshot(event: SnapshotEvent):
+        sizes.append(sum(isinstance(previous, SnapshotEvent) for previous in runner.bus.event_history.values()))
+        retained_completed.append(len(retired_ids.intersection(runner.bus.event_history)))
+        references.append(weakref.ref(event))
+
+    async def observe_completed(event: SnapshotCompletedEvent):
+        root = await runner.bus.find(SnapshotEvent, snapshot_id=event.snapshot_id, past=True, future=False)
+        assert root is not None
+        retired_ids.add(root.event_id)
+        retired_ids.update(item.event_id for item in await runner.bus.filter("*", child_of=root, past=True, future=False))
+
+    def release_timed_out_response(event: ProcessCompletedEvent):
+        if response_started.is_set() and event.hook_name == "on_Snapshot__30_chrome_navigate" and event.status == "failed":
+            release_response.set()
+
+    runner.bus.on("*", count_event)
+    runner.bus.on(SnapshotEvent, observe_snapshot)
+    runner.bus.on(SnapshotCompletedEvent, observe_completed)
+    runner.bus.on(ProcessCompletedEvent, release_timed_out_response)
+    try:
+        asyncio.run(runner.run())
+    finally:
+        release_response.set()
+    (tmp_path / "event-counts.json").write_text(json.dumps({"events": counts, "snapshots_in_history": sizes}))
+    snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("url"))
+    assert len(snapshots) == len(urls)
+    assert all(snapshot.status == "sealed" for snapshot in snapshots)
+    for index, snapshot in enumerate(snapshots):
+        if timeout_page and index == 3:
+            assert response_started.is_set()
+            result = ArchiveResult.objects.get(snapshot=snapshot, hook_name="on_Snapshot__30_chrome_navigate", status="failed")
+            assert "timeout" in result.output_str.lower()
+            continue
+        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", status="succeeded")
+        assert result.output_str == f"Saved page {index}"
+        assert (snapshot.output_dir / "title" / "title.txt").read_text().strip() == f"Saved page {index}"
+    assert sizes == [1] * len(urls), sizes
+    assert retained_completed == [0] * len(urls), retained_completed
+    gc.collect()
+    assert all(reference() is None for reference in references)

archivebox/tests/test_issue_1896.py

@@ -0,0 +1,193 @@
+"""Regression coverage for collection cookie paths and config edits on upgrade."""
+
+import json
+import sqlite3
+from contextlib import closing
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+from archivebox.config.configset import read_ini_config
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+@pytest.mark.parametrize("persona_name", ["Default", "default"])
+def test_add_reuses_persona_ignoring_case(initialized_archive, httpserver, persona_name):
+    """Either spelling reuses the same authenticated persona without moving files."""
+    run_archivebox_cmd(["persona", "create", "Default", "default"], cwd=initialized_archive, check=True)
+    persona_dir = initialized_archive / "personas" / "Default"
+    (persona_dir / "auth.json").touch()
+    (persona_dir / "cookies.txt").write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+    )
+    original_inode = (persona_dir / "cookies.txt").stat().st_ino
+    httpserver.expect_request("/").respond_with_data(
+        "<html><head><title>Existing persona capture</title></head><body>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script>'
+        "</body></html>",
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--persona", persona_name, "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert "issue1896=imported" in outputs[0].read_text()
+    assert (persona_dir / "cookies.txt").stat().st_ino == original_inode
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona ORDER BY name").fetchall() == [("Default",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_envs = db.execute("SELECT env FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_envs
+        assert all(json.loads(row[0])["ACTIVE_PERSONA"] == "Default" for row in hook_envs)
+
+
+@pytest.mark.parametrize("contents,explicit", [("", True), ("not JSON", True), ("not JSON", False)])
+def test_add_rejects_invalid_persona_auth_with_file_path(initialized_archive, httpserver, contents, explicit):
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.write_text(contents)
+    if explicit:
+        run_archivebox_cmd(
+            ["config", "--set", "AUTH_STORAGE_FILE=personas/Default/auth.json"],
+            cwd=initialized_archive,
+            check=True,
+        )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 1, result.stdout + result.stderr
+    logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    assert logs
+    assert f"Invalid JSON cookie export: {auth_file}" in "\n".join(log.read_text() for log in logs)
+    assert not list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert auth_file.read_text() == contents
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("isolation", ["crawl", "snapshot"])
+def test_add_with_empty_autodiscovered_persona_auth(initialized_archive, httpserver, cookie_file_exists, isolation):
+    """An empty auth placeholder must not block a crawl or hide Netscape cookies."""
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.touch()
+    cookies_file = persona_dir / "cookies.txt"
+    cookies_file.write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n" if cookie_file_exists else "",
+    )
+    httpserver.expect_request("/").respond_with_data(
+        '<html><title>Empty persona auth</title><body><output id="cookies"></output>'
+        '<script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false", "CHROME_ISOLATION": isolation},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert ("issue1896=imported" in outputs[0].read_text()) == cookie_file_exists
+    assert auth_file.read_bytes() == b""
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT cmd, env, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for cmd, env, exit_code in hook_rows:
+            hook_env = json.loads(env)
+            assert not hook_env.get("AUTH_STORAGE_FILE")
+            expected_cookies = cookies_file if isolation == "crawl" else Path(hook_env["PERSONAS_DIR"]) / "Default" / "cookies.txt"
+            assert hook_env["COOKIES_FILE"] == str(expected_cookies)
+            hook_name = Path(json.loads(cmd)[0]).name
+            # The crawl wait hook explicitly skips when each snapshot owns Chrome.
+            assert exit_code == (10 if isolation == "snapshot" and hook_name == "on_CrawlSetup__91_chrome_wait.js" else 0)
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("absolute_path", [False, True], ids=["collection-relative", "absolute"])
+def test_add_with_collection_relative_cookies(initialized_archive, httpserver, cookie_file_exists, absolute_path):
+    """Real hooks resolve relative paths against DATA_DIR and preserve absolute paths."""
+    cookies_file = (
+        initialized_archive.parent / f"{initialized_archive.name}-cookies.txt" if absolute_path else initialized_archive / "cookies.txt"
+    )
+    configured_path = str(cookies_file) if absolute_path else "cookies.txt"
+    if cookie_file_exists:
+        cookies_file.write_text(
+            "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+        )
+    httpserver.expect_request("/").respond_with_data(
+        "<html><title>Issue 1896</title><body><p>Collection cookie import regression: "
+        "this page must still be archived when the optional cookie file is absent.</p>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    run_archivebox_cmd(
+        ["config", "--set", f"COOKIES_FILE={configured_path}"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    html = outputs[0].read_text()
+    assert ("issue1896=imported" in html) == cookie_file_exists
+    launch_logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    logs = "\n".join(log.read_text() for log in launch_logs)
+    if cookie_file_exists:
+        assert f"from {cookies_file}" in logs
+    else:
+        assert "Cookies file not found" in logs
+        assert "continuing without cookie import" in logs
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT env, status, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for env, status, exit_code in hook_rows:
+            assert json.loads(env)["COOKIES_FILE"] == str(cookies_file)
+            assert (status, exit_code) == ("exited", 0)
+
+
+@pytest.mark.parametrize("enabled_key", ["USE_CHROME", "CHROME_ENABLED"])
+@pytest.mark.parametrize("startup_save", ["binary_sanitation", "host_refresh"])
+def test_init_preserves_edited_use_chrome_before_startup_save(initialized_archive, enabled_key, startup_save):
+    """Startup saves must not overwrite a newer file before config sync."""
+    config_path = initialized_archive / "ArchiveBox.conf"
+    config_path.write_text(config_path.read_text() + f"\n[SERVER_CONFIG]\n{enabled_key} = True\n")
+    run_archivebox_cmd(
+        ["config", "--set", f"WGET_BINARY={initialized_archive / 'removed-wget'}"]
+        if startup_save == "binary_sanitation"
+        else ["config", "--set", "TIMEOUT=37"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    if startup_save == "host_refresh":
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            db.execute("UPDATE machine_machine SET modified_at=?", [(datetime.now(timezone.utc) - timedelta(days=8)).isoformat()])
+            db.commit()
+    config_path.write_text(config_path.read_text().replace(f"{enabled_key} = True", f"{enabled_key} = False"))
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    run_archivebox_cmd(["init"], cwd=initialized_archive, timeout=120, check=True)
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    result = run_archivebox_cmd(["config", "--get", "CHROME_ENABLED"], cwd=initialized_archive)
+    assert result.returncode == 0, result.stderr
+    assert "CHROME_ENABLED = false" in result.stdout

archivebox/tests/test_persona_casing.py

@@ -0,0 +1,237 @@
+"""Persona names have the same identity on case-sensitive and insensitive disks."""
+
+import json
+import sqlite3
+import subprocess
+import sys
+from contextlib import closing
+
+import pytest
+from django.urls import reverse
+
+from archivebox.personas.models import Persona
+from archivebox.tests.conftest import ADMIN_TEST_HOST, api_client_request, get_free_port, run_archivebox_cmd, stop_server
+
+
+def test_cli_create_reuses_case_insensitive_persona(initialized_archive):
+    first = run_archivebox_cmd(["persona", "create", "--permissions=private", "personal"], cwd=initialized_archive, check=True)
+    second = run_archivebox_cmd(["persona", "create", "--permissions=public", "PERSONAL"], cwd=initialized_archive, check=True)
+    assert json.loads(second.stdout)["id"] == json.loads(first.stdout)["id"]
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        rows = db.execute("SELECT name, config FROM personas_persona WHERE name != 'Default'").fetchall()
+    assert len(rows) == 1
+    assert rows[0][0] == "personal"
+    assert json.loads(rows[0][1])["PERMISSIONS"] == "private"
+
+
+@pytest.mark.django_db
+def test_add_form_reuses_existing_lowercase_default(admin_client, admin_user):
+    original = Persona.objects.create(name="default", created_by=admin_user)
+    response = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    assert list(Persona.objects.values_list("id", "name")) == [(original.id, "default")]
+
+
+@pytest.mark.django_db
+def test_add_form_does_not_create_personas(admin_client):
+    assert not Persona.objects.exists()
+    response = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    assert not Persona.objects.exists()
+
+
+def test_init_creates_default_persona(initialized_archive):
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    assert (initialized_archive / "personas" / "Default" / "chrome_profile").is_dir()
+
+
+def test_server_initializes_default_persona_for_existing_collections(initialized_archive):
+    from archivebox.tests.conftest import cli_env, get_free_port, start_archivebox_server, stop_server
+
+    # Pre-fix collections may have no persona row until the first visit to /add/.
+    run_archivebox_cmd(
+        ["manage", "shell", "--no-imports", "-c", "from archivebox.personas.models import Persona; Persona.objects.all().delete()"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    port = get_free_port()
+    try:
+        start_archivebox_server(initialized_archive, port=port, env=cli_env(port=port, server=True))
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    finally:
+        stop_server(initialized_archive)
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_delete_preserves_files_and_recreation_adopts_directory_case(admin_client, admin_user):
+    persona = Persona.objects.create(name="personal", created_by=admin_user)
+    persona.ensure_dirs()
+    profile = persona.path / "chrome_profile" / "Preferences"
+    profile.write_text('{"profile":{"name":"Preserve me"}}')
+    cookies = persona.path / "cookies.txt"
+    cookies.write_text("# Netscape HTTP Cookie File\n")
+    before = (profile.stat().st_ino, cookies.stat().st_ino)
+    response = admin_client.post(reverse("admin:personas_persona_delete", args=[persona.pk]), {"post": "yes"}, HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 302
+    assert not Persona.objects.filter(pk=persona.pk).exists()
+    assert profile.read_text() == '{"profile":{"name":"Preserve me"}}'
+    response = admin_client.post(
+        reverse("admin:personas_persona_add"),
+        {
+            "name": "PERSONAL",
+            "created_by": str(admin_user.pk),
+            "permissions": "private",
+            "config": "{}",
+            "import_mode": "none",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302, response.content
+    recreated = Persona.objects.get()
+    assert recreated.name == "personal"
+    assert recreated.path == profile.parent.parent
+    assert (profile.stat().st_ino, cookies.stat().st_ino) == before
+
+
+@pytest.mark.django_db(transaction=True)
+def test_extension_sync_reuses_persona_case(client, api_headers):
+    original = Persona.objects.create(name="personal", config={"TIMEOUT": 37})
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": "case-test", "name": "PERSONAL", "settings": {}},
+        headers=api_headers,
+    )
+    assert response.status_code == 200, response.content
+    assert response.json()["created"] is False
+    assert response.json()["persona"]["id"] == str(original.pk)
+    assert Persona.objects.count() == 1
+    original.refresh_from_db()
+    assert original.config["TIMEOUT"] == 37
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_duplicate_name_identifies_existing_record_and_directory(admin_client, admin_user):
+    original = Persona.objects.create(name="personal", created_by=admin_user)
+    original.ensure_dirs()
+    response = admin_client.post(
+        reverse("admin:personas_persona_add"),
+        {
+            "name": "PERSONAL",
+            "created_by": str(admin_user.pk),
+            "permissions": "private",
+            "config": "{}",
+            "import_mode": "none",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 200
+    assert str(original.pk).encode() in response.content
+    assert str(original.path).encode() in response.content
+    assert list(Persona.objects.values_list("id", "name")) == [(original.id, "personal")]
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_case_only_edit_preserves_profile_directory(admin_client, admin_user):
+    persona = Persona.objects.create(name="personal", created_by=admin_user)
+    persona.ensure_dirs()
+    profile = persona.path / "chrome_profile" / "Preferences"
+    profile.write_text('{"profile":{"name":"Keep this profile"}}')
+    inode = profile.stat().st_ino
+    response = admin_client.post(
+        reverse("admin:personas_persona_change", args=[persona.pk]),
+        {"name": "PERSONAL", "created_by": str(admin_user.pk), "permissions": "private", "config": "{}", "_save": "Save"},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302, response.content
+    persona.refresh_from_db()
+    assert persona.name == "personal"
+    assert profile.stat().st_ino == inode
+    assert profile.read_text() == '{"profile":{"name":"Keep this profile"}}'
+
+
+@pytest.mark.django_db(transaction=True)
+def test_extension_sync_rejects_legacy_case_conflicts_before_writing(client, api_headers):
+    # Real rows representing collections created before case-insensitive matching.
+    Persona.objects.bulk_create([Persona(name="personal"), Persona(name="PERSONAL")])
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": "case-test-conflict", "name": "Personal", "settings": {}, "cookies_txt": "must not be written"},
+        headers=api_headers,
+    )
+    assert response.status_code == 409, response.content
+    assert "rename" in response.content.decode().lower()
+    assert "manually" in response.content.decode().lower()
+    assert Persona.objects.count() == 2
+    assert all(str(persona.pk).encode() in response.content for persona in Persona.objects.all())
+    assert all(not (persona.path / "cookies.txt").exists() for persona in Persona.objects.all())
+
+
+@pytest.fixture
+def case_sensitive_collection(tmp_path):
+    """Exercise actual Linux-style directory collisions, including on macOS."""
+    mount = tmp_path / "case-sensitive"
+    mount.mkdir()
+    if sys.platform == "darwin":
+        image = tmp_path / "personas.sparseimage"
+        subprocess.run(
+            ["hdiutil", "create", "-size", "128m", "-type", "SPARSE", "-fs", "Case-sensitive APFS", "-volname", "PersonaTest", str(image)],
+            check=True,
+            capture_output=True,
+        )
+        subprocess.run(["hdiutil", "attach", "-nobrowse", "-mountpoint", str(mount), str(image)], check=True, capture_output=True)
+    try:
+        yield mount
+    finally:
+        if sys.platform == "darwin":
+            subprocess.run(["hdiutil", "detach", str(mount)], check=True, capture_output=True)
+
+
+def test_cli_rejects_case_conflicting_directories_without_mutating(case_sensitive_collection):
+    collection = case_sensitive_collection
+    run_archivebox_cmd(["init", "--quick"], cwd=collection, check=True)
+    for name in ("Default", "default"):
+        profile = collection / "personas" / name / "chrome_profile"
+        profile.mkdir(parents=True, exist_ok=True)
+        (profile / "Preferences").write_text(name)
+    commands = (["persona", "create", "Default"], ["init", "--quick"], ["server", "--daemonize", f"127.0.0.1:{get_free_port()}"])
+    try:
+        for command in commands:
+            result = run_archivebox_cmd(command, cwd=collection)
+            assert result.returncode != 0
+            assert "rename" in result.stderr.lower()
+            assert "manually" in result.stderr.lower()
+            assert all(str(collection / "personas" / name) in result.stderr for name in ("Default", "default"))
+    finally:
+        stop_server(collection)
+    with closing(sqlite3.connect(collection / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    for name in ("Default", "default"):
+        assert (collection / "personas" / name / "chrome_profile" / "Preferences").read_text() == name
+
+
+def test_cli_adopts_existing_directory_spelling_on_case_sensitive_disk(case_sensitive_collection):
+    collection = case_sensitive_collection
+    run_archivebox_cmd(["init", "--quick"], cwd=collection, check=True)
+    profile = collection / "personas" / "personal" / "chrome_profile" / "Preferences"
+    profile.parent.mkdir(parents=True)
+    profile.write_text('{"profile":{"name":"Keep me"}}')
+    inode = profile.stat().st_ino
+    run_archivebox_cmd(["persona", "create", "PERSONAL"], cwd=collection, check=True)
+    with closing(sqlite3.connect(collection / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona ORDER BY name").fetchall() == [("Default",), ("personal",)]
+        # A legacy database can disagree with an existing directory's spelling.
+        db.execute("UPDATE personas_persona SET name='PERSONAL' WHERE name='personal'")
+        db.commit()
+    result = run_archivebox_cmd(["persona", "create", "Personal"], cwd=collection, check=True)
+    assert json.loads(result.stdout)["name"] == "personal"
+    assert not (collection / "personas" / "PERSONAL").exists()
+    assert profile.stat().st_ino == inode
+    assert profile.read_text() == '{"profile":{"name":"Keep me"}}'

archivebox/tests/test_release_workflow.py

@@ -185,13 +185,22 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
 
     assert all(step.get("name") != "Verify published PyPI package installs and runs" for step in python_release["steps"])
     ci = yaml.safe_load(CI_WORKFLOW.read_text())
-    screenshots = yaml.safe_load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text())
+    screenshots = yaml.load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text(), Loader=yaml.BaseLoader)
     prepare_path = ci["jobs"]["prepare"]["uses"]
-    assert screenshots["jobs"]["prepare"]["uses"] == prepare_path
     prepare = yaml.safe_load((REPO_ROOT / prepare_path).read_text())
     assert prepare["jobs"]["prepare"]["steps"][-1]["uses"] == "ArchiveBox/monorepo/.github/actions/prepare-release-version@main"
-    assert screenshots["jobs"]["deploy"]["needs"] == "prepare"
-    assert screenshots["jobs"]["deploy"]["if"] == "needs.prepare.outputs.run_jobs == 'true'"
+    assert set(screenshots["on"]) == {"schedule", "workflow_dispatch"}
+    _, hours, *days = screenshots["on"]["schedule"][0]["cron"].split()
+    assert days == ["*", "*", "*"]
+    assert len(hours.split(",")) == 2
+    first, second = map(int, hours.split(","))
+    assert second - first == 12
+    assert "prepare" not in screenshots["jobs"]
+    assert screenshots["jobs"]["changes"]["steps"][0]["uses"] == "ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main"
+    assert screenshots["jobs"]["deploy"]["needs"] == "changes"
+    assert screenshots["jobs"]["deploy"]["if"] == "needs.changes.outputs.changed == 'true'"
+    assert screenshots["jobs"]["complete"]["needs"] == ["changes", "deploy"]
+    assert screenshots["jobs"]["complete"]["name"] == "Scheduled inputs: ${{ needs.changes.outputs.key }}"
     for name, job in ci["jobs"].items():
         if name not in {"prepare", "required"}:
             assert job["needs"] == "prepare"

archivebox/tests/test_runner_deletion.py

@@ -0,0 +1,177 @@
+"""Exercise queued deletion in the existing runner with a real server."""
+
+import json
+import textwrap
+import time
+from pathlib import Path
+
+import pytest
+
+from archivebox.tests.conftest import (
+    cli_env,
+    get_free_port,
+    kill_processes_for_data_dir,
+    run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    wait_for_log,
+)
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("restart", [False, True])
+def test_runner_retries_failed_deletion_without_blocking_captures(initialized_archive, recursive_test_site, restart):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+
+    port = get_free_port()
+    env = cli_env(
+        live=True,
+        server=True,
+        port=port,
+        PLUGINS="title",
+        SAVE_TITLE="True",
+        SEARCH_BACKEND_ENGINE="ripgrep",
+    )
+    prepared = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            "import json; from archivebox.crawls.models import Crawl; "
+            "from archivebox.core.models import Snapshot, ArchiveResult; "
+            "from archivebox.base_models.models import get_or_create_system_user_pk; "
+            "c=Crawl.objects.create(urls='https://example.com/delete',created_by_id=get_or_create_system_user_pk(),status='sealed'); "
+            "s=Snapshot.objects.create(url='https://example.com/delete',crawl=c,status='deleting'); "
+            "p=s.output_dir/'wget'; p.mkdir(parents=True,exist_ok=True); "
+            "(p/'index.html').write_text('keep this until cleanup succeeds'); "
+            "ArchiveResult.objects.create(snapshot=s,plugin='wget',hook_name='on_Snapshot__06_wget',status='succeeded'); "
+            "p.chmod(0o500); print(json.dumps({'id':str(s.pk),'path':str(p)}))",
+        ],
+        cwd=initialized_archive,
+        env=env,
+    )
+    assert prepared.returncode == 0, prepared.stderr or prepared.stdout
+    fixture = json.loads(prepared.stdout.strip().splitlines()[-1])
+    protected = Path(fixture["path"])
+    worker_log = initialized_archive / "logs" / "worker_runner.log"
+    server = None
+    try:
+        server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-worker-server.log")
+        wait_for_log(worker_log, "PermissionError", timeout=30)
+        with use_archivebox_db(initialized_archive):
+            assert Process.objects.filter(worker_type="worker_runner", status="running").exists()
+            assert not Process.objects.filter(worker_type="worker_snapshot_delete").exists()
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+            assert ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert (protected / "index.html").is_file()
+        added = run_archivebox_cmd(
+            ["add", "--bg", "--plugins=title", recursive_test_site["root_url"]],
+            cwd=initialized_archive,
+            env=env,
+        )
+        assert added.returncode == 0, added.stderr or added.stdout
+        deadline = time.monotonic() + 60
+        with use_archivebox_db(initialized_archive):
+            while time.monotonic() < deadline:
+                captured = Snapshot.objects.filter(url=recursive_test_site["root_url"], status="sealed").first()
+                if captured is not None:
+                    break
+                assert server.poll() is None
+                time.sleep(0.1)
+            else:
+                pytest.fail((initialized_archive / "logs" / "worker_runner.log").read_text()[-12000:])
+            result = ArchiveResult.objects.get(snapshot=captured, plugin="title", status="succeeded")
+            assert result.output_files
+            assert captured.title == "Root"
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+        if restart:
+            stop_archivebox_process(server)
+            server = None
+        protected.chmod(0o700)
+        if restart:
+            server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-restarted-server.log")
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(initialized_archive):
+            while Snapshot.objects.filter(pk=fixture["id"]).exists() and time.monotonic() < deadline:
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=fixture["id"]).exists(), worker_log.read_text()
+            assert not ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert not protected.parent.exists()
+    finally:
+        if protected.exists():
+            protected.chmod(0o700)
+        if server is not None:
+            stop_archivebox_process(server)
+        kill_processes_for_data_dir(initialized_archive)
+
+
+def test_deletion_releases_database_before_filesystem_io(initialized_archive):
+    result = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            textwrap.dedent("""
+                import json
+                import sqlite3
+                import sys
+                from pathlib import Path
+                from django.db import connection
+                from archivebox.base_models.models import get_or_create_system_user_pk
+                from archivebox.core.models import ArchiveResult, Snapshot
+                from archivebox.crawls.models import Crawl
+                from archivebox.services.runner import run_pending_crawls
+
+                # DELETE mode exposes both read-cursor and transaction locks.
+                with connection.cursor() as cursor:
+                    cursor.execute('PRAGMA journal_mode=DELETE')
+                    assert cursor.fetchone()[0] == 'delete'
+                crawl = Crawl.objects.create(
+                    urls='https://example.com/delete',
+                    created_by_id=get_or_create_system_user_pk(), status='sealed',
+                )
+                paths = set()
+                snapshot_ids = []
+                for i in range(2):
+                    snapshot = Snapshot.objects.create(url=f'https://example.com/delete/{i}', crawl=crawl, status='deleting')
+                    path = snapshot.output_dir
+                    path.mkdir(parents=True, exist_ok=True)
+                    (path / 'saved.html').write_text('real saved output')
+                    ArchiveResult.objects.create(snapshot=snapshot, plugin='wget', status='succeeded')
+                    paths.add(str(path))
+                    snapshot_ids.append(snapshot.pk)
+                checked = set()
+
+                def inspect_filesystem_operation(event, args):
+                    if event != 'shutil.rmtree' or str(args[0]) not in paths:
+                        return
+                    assert not connection.in_atomic_block
+                    # Observe the real rmtree call without replacing it. An
+                    # independent writer must acquire its lock immediately.
+                    writer = sqlite3.connect(connection.settings_dict['NAME'], timeout=0, isolation_level=None)
+                    try:
+                        writer.execute('BEGIN EXCLUSIVE')
+                        writer.execute('UPDATE crawls_crawl SET notes=? WHERE id=?', ('concurrent write succeeded', crawl.pk.hex))
+                        writer.execute('COMMIT')
+                    finally:
+                        writer.close()
+                    checked.add(str(args[0]))
+
+                sys.addaudithook(inspect_filesystem_operation)
+                run_pending_crawls(maintenance_only=True)
+                assert checked == paths
+                assert not Snapshot.objects.filter(crawl=crawl).exists()
+                assert not ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids).exists()
+                assert all(not Path(path).exists() for path in paths)
+                crawl.refresh_from_db()
+                assert crawl.notes == 'concurrent write succeeded'
+                print(json.dumps({'verified_deletions': len(checked)}))
+            """),
+        ],
+        cwd=initialized_archive,
+        env=cli_env(disable_extractors=True),
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert json.loads(result.stdout.strip().splitlines()[-1]) == {"verified_deletions": 2}

archivebox/tests/test_takeover_util_2.py

@@ -175,6 +175,10 @@ def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive
     worker_text = worker_config.read_text(encoding="utf-8")
     command = next(line.partition("=")[2] for line in worker_text.splitlines() if line.startswith("command="))
     assert shlex.split(command)[0] == str(sonic_binary), worker_text
+    supervisor_pids = re.findall(r"Supervisord connected \(pid=(\d+)\)", result.stderr)
+    assert supervisor_pids, result.stderr
+    assert all(not pid_is_alive(int(pid)) for pid in supervisor_pids), "Search CLI returned while its owned supervisord was still alive"
+    assert result.stdout == "", result.stdout
     from archivebox.workers.supervisord_util import _sonic_worker_bind_target
 
     assert _sonic_worker_bind_target({"name": "worker_sonic", "command": command}) == ("127.0.0.1", sonic_port)
@@ -198,6 +202,7 @@ def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive
         timeout=60,
     )
     assert relative_result.returncode == 0, relative_result.stderr or relative_result.stdout
+    assert relative_result.stdout == "", relative_result.stdout
     relative_worker_text = worker_config.read_text(encoding="utf-8")
     relative_command = next(line.partition("=")[2] for line in relative_worker_text.splitlines() if line.startswith("command="))
     assert Path(shlex.split(relative_command)[0]).name == relative_binary.name, relative_worker_text

archivebox/tests/test_ui_add_view.py

@@ -37,6 +37,12 @@ ADD_FORM_DEFAULTS = {
 }
 
 
+@pytest.fixture(autouse=True)
+def initialized_persona(db):
+    # Match init/server startup; rendering /add/ must no longer create DB rows.
+    return Persona.get_or_create_default()
+
+
 @pytest.fixture
 def admin_user(db):
     return User.objects.create_superuser(

archivebox/tests/test_ui_admin_crawl.py

@@ -343,6 +343,7 @@ def test_discovered_snapshots_inherit_current_crawl_tags(crawl):
     assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl", "discovered"}
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, admin_user):
     crawl = Crawl.objects.create(
         urls="https://example.com/remove-me",
@@ -352,6 +353,9 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
         crawl=crawl,
         url="https://example.com/remove-me",
     )
+    output_dir = snapshot.output_dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "captured.txt").write_text("remove this capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -362,11 +366,13 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
     assert response.status_code == 200
     assert response.json()["ok"] is True
     assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
+    assert not output_dir.exists()
 
     crawl.refresh_from_db()
     assert "https://example.com/remove-me" not in crawl.urls
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(client, admin_user):
     crawl = Crawl.objects.create(
         urls=("https://cdn.example.com/asset.js\nhttps://cdn.example.com/second.js\nhttps://example.com/root"),
@@ -382,6 +388,12 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
         url="https://example.com/root",
         status=Snapshot.StatusChoices.SEALED,
     )
+    removed_dir = queued_snapshot.output_dir
+    removed_dir.mkdir(parents=True, exist_ok=True)
+    (removed_dir / "captured.txt").write_text("excluded capture")
+    preserved_file = preserved_snapshot.output_dir / "captured.txt"
+    preserved_file.parent.mkdir(parents=True, exist_ok=True)
+    preserved_file.write_text("preserved capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -401,6 +413,8 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
     assert "https://example.com/root" in crawl.urls
     assert not Snapshot.objects.filter(pk=queued_snapshot.pk).exists()
     assert Snapshot.objects.filter(pk=preserved_snapshot.pk).exists()
+    assert not removed_dir.exists()
+    assert preserved_file.read_text() == "preserved capture"
 
 
 def test_snapshot_from_json_trims_markdown_suffixes_on_discovered_urls(crawl):

archivebox/tests/test_ui_admin_snapshot.py

@@ -26,6 +26,65 @@ pytestmark = pytest.mark.django_db(transaction=True)
 REPO_ROOT = Path(__file__).resolve().parents[2]
 
 
+def test_bulk_snapshot_delete_keeps_rows_until_worker_verifies_files(client, admin_user, crawl):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.runner import run_pending_crawls
+
+    snapshots = [Snapshot.objects.create(url=f"https://example.com/delete/{i}", crawl=crawl) for i in range(2)]
+    output_dirs = [snapshot.output_dir for snapshot in snapshots]
+    for snapshot, output_dir in zip(snapshots, output_dirs):
+        (output_dir / "wget").mkdir(parents=True, exist_ok=True)
+        (output_dir / "wget" / "index.html").write_text("real saved output")
+        ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    ids = [snapshot.pk for snapshot in snapshots]
+    client.force_login(admin_user)
+    response = client.post(
+        reverse("admin:core_snapshot_changelist"),
+        {"action": "delete_snapshots", ACTION_CHECKBOX_NAME: [str(pk) for pk in ids]},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302
+    assert Snapshot.objects.filter(pk__in=ids, status="deleting").count() == 2
+    assert ArchiveResult.objects.filter(snapshot_id__in=ids).count() == 2
+    assert all((path / "wget" / "index.html").is_file() for path in output_dirs)
+    pending = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST)
+    assert pending.status_code == 200
+    assert "Deleting" in pending.content.decode()
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk__in=ids).exists()
+    assert not ArchiveResult.objects.filter(snapshot_id__in=ids).exists()
+    assert not any(path.exists() for path in output_dirs)
+
+
+def test_snapshot_delete_preserves_rows_when_real_filesystem_cleanup_fails(snapshot):
+    from archivebox.core.models import ArchiveResult, Snapshot
+
+    output_dir = snapshot.output_dir
+    protected = output_dir / "wget"
+    protected.mkdir(parents=True, exist_ok=True)
+    saved_file = protected / "index.html"
+    saved_file.write_text("must not lose the row before this file is removed")
+    result = ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    snapshot_id = snapshot.pk
+    protected.chmod(0o500)
+    try:
+        with pytest.raises(PermissionError):
+            Snapshot.objects.filter(pk=snapshot_id).delete()
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert ArchiveResult.objects.filter(pk=result.pk).exists()
+        assert saved_file.is_file()
+        from archivebox.services.runner import run_pending_crawls
+
+        run_pending_crawls(maintenance_only=True)
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert saved_file.is_file()
+    finally:
+        protected.chmod(0o700)
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+    assert not output_dir.exists()
+
+
 def test_failed_hook_partial_file_does_not_become_successful_card(snapshot, cached_abxpkg_lib_dir):
     partial = Path(snapshot.output_dir) / "title" / "partial.html"
     partial.parent.mkdir(parents=True, exist_ok=True)

archivebox/tests/test_ui_config_views.py

@@ -7,7 +7,7 @@ from archivebox.plugins import views as plugin_views
 from archivebox.core import views as core_views
 from archivebox.config import CONSTANTS
 from archivebox.machine.models import Machine
-from archivebox.tests.conftest import install_real_binary
+from archivebox.tests.conftest import ADMIN_TEST_HOST, install_real_binary
 
 
 pytestmark = pytest.mark.django_db
@@ -75,18 +75,42 @@ def test_binaries_list_view_uses_db_version_and_hides_youtube_dl_alias(admin_req
 
     assert len(context["table"]["Binary Name"]) == 1
     assert str(context["table"]["Binary Name"][0].link_item) == "yt-dlp"
-    assert context["table"]["Found Version"][0] == f"✅ {binary.version}"
+    assert context["table"]["Recorded Version"][0] == binary.version
     assert context["table"]["Provided By"][0] == binary.binprovider
-    assert context["table"]["Found Abspath"][0] == binary.abspath
+    assert context["table"]["Recorded Abspath"][0] == binary.abspath
 
 
 def test_binaries_list_view_only_shows_persisted_records(admin_request):
     context = config_views.binaries_list_view.__wrapped__(admin_request("/admin/environment/binaries/"))
 
     assert context["table"]["Binary Name"] == []
-    assert context["table"]["Found Version"] == []
+    assert context["table"]["Recorded Version"] == []
     assert context["table"]["Provided By"] == []
-    assert context["table"]["Found Abspath"] == []
+    assert context["table"]["Recorded Abspath"] == []
+
+
+@pytest.mark.django_db(transaction=True)
+def test_binaries_page_labels_removed_installation_as_recorded_metadata(admin_client, machine, hermetic_lib_dir):
+    from pathlib import Path
+
+    binary = install_real_binary("node", machine=machine)
+    binary_path = Path(binary.abspath)
+    assert binary_path.is_relative_to(hermetic_lib_dir)
+    assert binary_path.is_symlink()
+    binary_path.unlink()
+    assert not binary_path.exists()
+
+    response = admin_client.get("/admin/environment/binaries/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    html = response.content.decode()
+    assert "/admin/environment/binaries/node/" in html
+    assert "Recorded Version" in html
+    assert "Recorded Abspath" in html
+    assert binary.version in html
+    assert str(binary_path) in html
+    assert f"✅ {binary.version}" not in html
+    binary.refresh_from_db()
+    assert binary.abspath == str(binary_path)
 
 
 @pytest.mark.django_db(transaction=True)

bin/lychee_preprocess.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env -S uv run --no-project --script
+import pathlib
+import re
+import sys
+from urllib.parse import unquote, urlsplit
+
+p = pathlib.Path(sys.argv[1])
+text = p.read_text()
+if p.suffix not in (".md", ".html", ".rst"):
+    # Preserve source line numbers, but feed only literal documentation URLs to Lychee.
+    pattern = r'https?://(?:github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki|/blob/|/?#)|raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/|docs\.archivebox\.io)[^\s<>\)\]"\x27`\\\[]*'
+    text = "\n".join(" ".join(u for u in re.findall(pattern, line) if not re.search(r"[{}]", u)) for line in text.splitlines())
+if p.suffix == ".md" and p.parent.name == "docs":
+    pages = {page.stem.lower(): page.stem for page in p.parent.glob("*.md")}
+
+    def wikilink(match):
+        label, _, target = match[1].partition("|")
+        page, sep, fragment = (target or label).replace(" ", "-").partition("#")
+        page = pages.get(page.lower(), page)
+        return f"[{label}]({page}.md{sep}{fragment})"
+
+    text = re.sub(r"\[\[([^]\n]+)\]\]", wikilink, text)
+if p.suffix == ".md":
+    # Lychee folds local Markdown fragments to lowercase; GitHub heading IDs are lowercase.
+    # Preserve explicitly declared mixed-case HTML IDs, but reject heading case typos.
+    errors = []
+    for match in re.finditer(r'\]\(([^\s)]+)\)|href=["\']([^"\']+)', text):
+        url = urlsplit(match[1] or match[2])
+        fragment = unquote(url.fragment)
+        if url.scheme or url.netloc or fragment == fragment.lower():
+            continue
+        target = p.parent / unquote(url.path) if url.path else p
+        if not target.suffix:
+            target = target.with_suffix(".md")
+        if target.suffix != ".md" or not target.is_file():
+            continue  # Lychee handles missing files and non-Markdown targets.
+        ids = re.findall(r'(?:id|name)=["\']([^"\']+)', target.read_text())
+        if fragment not in ids:
+            line = text[: match.start()].count("\n") + 1
+            errors.append(f"{p}:{line}: heading fragment #{fragment} must match lowercase GitHub IDs")
+    if errors:
+        sys.exit("\n".join(errors))
+print(text)

docs/Configuration.md

@@ -138,7 +138,7 @@ archivebox add --persona=personal https://members.example.com/feed
 **Possible Values:** [`Default`]/`personal`/`work`/...
 The persona profile used when no explicit persona is selected for a new crawl. The selected persona is stored on the Crawl row; `DEFAULT_PERSONA` is not duplicated into `Crawl.config`. Personas bundle a Chrome user-data-dir, a `cookies.txt`, auth state, a user-agent, and any other per-identity config into a single named profile, letting you swap between archiving contexts (logged-out vs. signed-into-work-account vs. signed-into-personal-account) without manually juggling files.
 
-ArchiveBox auto-creates the named persona on disk if it doesn't already exist. See the [Personas wiki page](https://github.com/ArchiveBox/ArchiveBox/wiki/Personas) for the full directory layout.
+ArchiveBox auto-creates the named persona on disk if it doesn't already exist. See the [Chromium profile setup guide](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile) for profile setup instructions.
 
 *Related options:*
 [`COOKIES_FILE`](#cookies_file)
@@ -588,7 +588,7 @@ When `True`, every LDAP user who successfully authenticates is auto-promoted to
 *Options for the on-disk layout, file permissions, and temp/lib directories that ArchiveBox reads and writes during archiving.*
 
 > [!NOTE]
-> Docker-only [`PUID` and `PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid) are entrypoint environment variables, not ArchiveBox configuration keys.
+> Docker-only [`PUID` and `PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues) are entrypoint environment variables, not ArchiveBox configuration keys.
 
 ---
 <a id="dir_output_permissions"></a>
@@ -664,7 +664,7 @@ ArchiveBox stores all of its index metadata in a single SQLite database file (`i
 > These are advanced operator tuning options. If you are not actively diagnosing `database is locked` errors or planning a non-default storage layout, you can safely leave everything in this section at its default.
 
 *Learn more:*
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#sqlite-database-is-locked
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database-and-filesystem-contention-issues-when-running-multiple-archivebox-processes
 - https://www.sqlite.org/wal.html
 - https://www.sqlite.org/pragma.html
 

docs/Docker.md

@@ -8,16 +8,16 @@ Running ArchiveBox with Docker allows you to manage it in a container without ex
 
 <img src="https://imgur.zervice.io/qFAPRwC.png" width="20%" align="right"/>
 
-- [Overview](#Overview)
+- [Overview](#overview)
 - [Docker Compose](#docker-compose) ⭐️ (recommended)
   - [Setup](#setup)
-  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-docker-compose-%EF%B8%8F)
+  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-docker-compose-%EF%B8%8F)
   - [Usage](#usage)
   - [Accessing the data](#accessing-the-data)
   - [Configuration](#configuration)
 - [Plain Docker](#docker)
   - [Setup](#setup-1)
-  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-plain-docker)
+  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-plain-docker)
   - [Usage](#usage-1)
   - [Accessing the data](#accessing-the-data-1)
   - [Configuration](#configuration-1)
@@ -82,7 +82,7 @@ docker compose exec archivebox archivebox config --set SEARCH_BACKEND_ENGINE=rip
 
 ### Upgrading
 
-See the wiki page on [Upgrading or Merging Archives: Upgrading with Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-docker-compose-%EF%B8%8F) for instructions. ➡️
+See the wiki page on [Upgrading or Merging Archives: Upgrading with Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-docker-compose-%EF%B8%8F) for instructions. ➡️
 
 <br/>
 
@@ -220,13 +220,13 @@ Then open `/admin/` on the hostname or IP used to reach ArchiveBox (local exampl
 
 *(You can create a collection in any directory you want, `~/archivebox/data` is just used as an example here)*
 
-If you encounter permissions issues, make sure the mounted data directory is writable by its intended owner. Docker startup uses explicit [`PUID`/`PGID`](#puid--pgid) values when supplied, otherwise the first non-root owner detected from the existing collection or the default `archivebox` user when the data directory is root-owned.
+If you encounter permissions issues, make sure the mounted data directory is writable by its intended owner. Docker startup uses explicit [`PUID`/`PGID`](#docker-puid--pgid) values when supplied, otherwise the first non-root owner detected from the existing collection or the default `archivebox` user when the data directory is root-owned.
 
 <br/>
 
 ### Upgrading
 
-See the wiki page on [Upgrading or Merging Archives: Upgrading with plain Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-plain-docker) for instructions. ➡️
+See the wiki page on [Upgrading or Merging Archives: Upgrading with plain Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-plain-docker) for instructions. ➡️
 
 <br/>
 

docs/Install.md

@@ -11,8 +11,8 @@ ArchiveBox is primarily distributed as a Python package installed with `uv`, but
    - **[Option A. Docker / Docker Compose ⭐️](#option-a-docker--docker-compose-setup-%EF%B8%8F)**
    - [Option B. uv Install Shortcut](#option-b-uv-install-shortcut)
    - [Option C. System Package Manager Setup](#option-c-bare-metal-setup)
-     - *[Upgrading ArchiveBox to a new version](#upgrading-archivebox-to-a-new-version)*
- - *[Next Steps](#next-steps)*
+     - *[Upgrading ArchiveBox to a new version](#next-steps-upgrading-archivebox-to-a-new-version)*
+ - *[Next Steps](#next-steps-add-some-urls-to-archive-and-try-out-cli--web-ui)*
 
 
 ## Supported Systems
@@ -99,7 +99,7 @@ See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) do
 <img src="https://avatars0.githubusercontent.com/u/1503512?s=200&v=4" width="100px" align="right"/>
 
 **More info:**
- - For help installing these, see the [Manual Setup](#manual-setup), [[Troubleshooting]] and [[Chromium Install]] pages.
+ - For help installing these, see the [Manual Setup](#option-c-bare-metal-setup), [[Troubleshooting]] and [[Chromium Install]] pages.
  - For per-plugin binary and enable/disable options (CHROME_BINARY, RIPGREP_BINARY, `<plugin>_ENABLED`, etc.) see the [abx-plugins config reference](https://plugins.archivebox.io/).
 
 

docs/Release-Notes-v0.9.md

@@ -148,7 +148,7 @@ This is intentionally limited to advisory threads where the maintainer explicitl
 
 - [ArchiveBox documentation](https://docs.archivebox.io/)
 - [ArchiveBox browser extension](https://github.com/ArchiveBox/archivebox-browser-extension)
-- [ArchiveBox REST API documentation](https://docs.archivebox.io/dev/#rest-api)
+- [ArchiveBox REST API documentation](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.api.html)
 - [ArchiveBox releases](https://github.com/ArchiveBox/ArchiveBox/releases)
 - [Full source comparison from v0.7.4 to the current v0.9 release candidate](https://github.com/ArchiveBox/ArchiveBox/compare/v0.7.4...v0.9.35rc410)
 

docs/Security-Overview.md

@@ -46,7 +46,7 @@ archivebox persona create --import=chrome personal
 archivebox add --persona=personal 'https://members.example.com/'
 ```
 
-To get started, import a dedicated browser profile into a [persona](https://github.com/ArchiveBox/ArchiveBox/wiki/Personas). A persona keeps its Chrome profile and `cookies.txt` together and applies the same identity consistently across extractors.
+To get started, import a dedicated browser profile into a [persona](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile). A persona keeps its Chrome profile and `cookies.txt` together and applies the same identity consistently across extractors.
 
 ➡️ For full instructions on setting up a Chromium user profile see here: https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile
 
@@ -134,9 +134,9 @@ The ArchiveBox database is an unencrypted, uncompressed SQLite3 `index.sqlite3`
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#modify-the-archivebox-sqlite3-db-directly
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#example-adding-a-new-user-with-a-hashed-password
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication#username--password-the-default
 
 ### Filesystem
 
@@ -150,11 +150,11 @@ How much are you planning to archive? Only a few bookmarked articles, or thousan
 
 Consider what permissioning to apply to your archive folder carefully. Limit access to the fewest possible users by checking folder ownership and setting [`OUTPUT_PERMISSIONS`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#output_permissions) accordingly. Generally the `index.sqlite3` file, `archive/` folder, and `ArchiveBox.conf` file must all be owned and writable by the `archivebox` user or a dedicated non-root user.
 
-When running with Docker, the entrypoint uses explicit [`PUID`/`PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid) values when supplied, otherwise the existing non-root owner of the mounted data directory or the image's `archivebox` user.
+When running with Docker, the entrypoint uses explicit [`PUID`/`PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues) values when supplied, otherwise the existing non-root owner of the mounted data directory or the image's `archivebox` user.
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout
 - https://github.com/ArchiveBox/ArchiveBox#output-formats
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#filesystem-doesnt-support-fsync-eg-network-mounts
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts
 - https://github.com/ArchiveBox/ArchiveBox#storage-requirements

docs/Setting-Up-Storage.md

@@ -18,7 +18,7 @@ services:
 <h4>Related Docs</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox#archive-layout">README: Archive Layout</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>

docs/Setting-up-Authentication.md

@@ -91,7 +91,7 @@ LDAP_LASTNAME_ATTR="sn"
 LDAP_EMAIL_ATTR="mail"
 ```
 
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#ldap
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#ldap-settings
 - https://github.com/ArchiveBox/ArchiveBox/pull/1214
 - https://github.com/django-auth-ldap/django-auth-ldap#example-configuration
 - https://jumpcloud.com/blog/what-is-ldap-authentication

docs/Troubleshooting.md

@@ -4,11 +4,11 @@
 
 What are you having an issue with?:
 
-- [Installing ArchiveBox](#Installing)
+- [Installing ArchiveBox](#installing)
 - [Upgrading ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives)
 - [Configuring ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
-- [Archiving content with ArchiveBox](#Archiving)
-- [Hosting your collection publicly](#Hosting-the-Archive)
+- [Archiving content with ArchiveBox](#archiving)
+- [Hosting your collection publicly](#hosting-the-archive)
 - [Database and filesystem issues](#database)
 
 ---
@@ -296,5 +296,5 @@ See here for more info:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Merging-Collections
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder

docs/Upgrading.md

@@ -61,7 +61,7 @@ As of v0.4 and above, ArchiveBox uses the Django migrations system for determini
 
 More info:
 - https://docs.djangoproject.com/en/4.0/topics/migrations/
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-migrations-errors-or-upgrade-issues
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database-migrations-errors-or-upgrade-issues
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
 
 ---
@@ -146,7 +146,7 @@ archivebox status      # check that everything succeeded
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox#-package-manager-setup
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 - https://github.com/ArchiveBox/homebrew-archivebox
 - https://github.com/ArchiveBox/debian-archivebox
 - https://aur.archlinux.org/packages/archivebox

docs/Usage.md

@@ -6,10 +6,10 @@
 
 <img src="https://imgur.zervice.io/aQZZcku.png" width="20%" align="right"/>
 
-- [CLI Usage](#CLI-Usage): Docs and examples for the ArchiveBox command line interface.
-- [Admin UI Usage](#UI-Usage): Docs and screenshots for the outputted HTML archive interface.
-- [Browser Extension Usage](#Browser-Extension-Usage): Docs and screenshots for the outputted HTML archive interface.
-- [Disk Layout](#Disk-Layout): Description of the archive folder structure and contents.
+- [CLI Usage](#cli-usage): Docs and examples for the ArchiveBox command line interface.
+- [Admin UI Usage](#ui-usage): Docs and screenshots for the outputted HTML archive interface.
+- [Browser Extension Usage](#browser-extension-usage): Docs and screenshots for the outputted HTML archive interface.
+- [Disk Layout](#disk-layout): Description of the archive folder structure and contents.
 
 **Related:**
 
@@ -36,10 +36,10 @@ You can share a single archivebox data directory between Docker and non-Docker i
 
 For more examples see [README: Usage](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-cli-usage) and [[Docker]] pages.
 
-- [Run ArchiveBox with configuration options](#Run-ArchiveBox-with-configuration-options)
-- [Import a single URL](#Import-a-single-URL)
-- [Import a list of URLs from a text file](#Import-a-list-of-URLs-from-a-text-file)
-- [Import list of links from browser history](#Import-list-of-links-from-browser-history)
+- [Run ArchiveBox with configuration options](#run-archivebox-with-configuration-options)
+- [Import a single URL](#import-a-single-url)
+- [Import a list of URLs from a text file](#import-a-list-of-urls-from-a-text-file)
+- [Import list of links from browser history](#import-list-of-links-from-browser-history)
 
 ---
 
@@ -146,7 +146,7 @@ archivebox server 0.0.0.0:5797     # start the archivebox web server
 
 Open <http://admin.archivebox.localhost:5797/admin/> in a browser to create the first admin and finish web setup. To create additional accounts from the CLI, use `archivebox manage createsuperuser`.
 
-*See the [Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions) and [Security Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#archiving-private-content) for more info...*
+*See the [Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions) and [Security Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#archiving-content-behind-log-ins--advanced-users-only) for more info...*
 
 Or if you prefer to generate a [static HTML index](https://github.com/ArchiveBox/ArchiveBox#static-archive-exporting) instead of using the built-in web server, you can run `archivebox list --html --with-headers > ./index.html` and then open `./index.html` in a browser.  You should see something [like this](https://demo.archivebox.io).
 
@@ -254,8 +254,8 @@ Simply back up the entire `data/` folder to back up your archive, e.g. `zip -r d
 For more info about ArchiveBox's database/filesystem layout and troubleshooting steps:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#modify-the-archivebox-sqlite3-db-directly
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
 
 
 ### Large Archives
@@ -280,7 +280,7 @@ which are already in the index.
 
 For more info about troubleshooting filesystem permissions, performance, or issues when running on a NAS:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
 
 <br/>
 
@@ -302,9 +302,9 @@ UPDATE auth_user SET email = 'someNewEmail@example.com' WHERE username = 'someUs
 ```
 
 More info:
-- https://github.com/ArchiveBox/ArchiveBox#-sqlpythonfilesystem-usage
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#modify-the-archivebox-sqlite3-db-directly
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
+- https://github.com/ArchiveBox/ArchiveBox#usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
 - https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
 - https://adamobeng.com/wddbfs-mount-a-sqlite-database-as-a-filesystem/
 
@@ -341,7 +341,7 @@ $ archivebox shell
 ```
 
 For more info and example usage:
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#example-adding-a-new-user-with-a-hashed-password
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication#username--password-the-default
 - https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/cli/
 - https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/common.py
 - https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py

docs/Web-Archiving-Community.md

@@ -16,10 +16,10 @@ Whether you want to learn which organizations are the big players in the web arc
 
 <img src="https://imgur.zervice.io/duS8Lm7.png" width="200px" align="right" style="float: right; margin: 5px"/>
 
-- [The Master Lists](#The-Master-Lists)  
+- [The Master Lists](#the-master-lists)  
   *Community-maintained indexes of web archiving tools and groups by IIPC, COPTR, ArchiveTeam, Wikipedia, & the ASA.* 
 
-- [Web Archiving Software](#Web-Archiving-Projects)  
+- [Web Archiving Software](#web-archiving-projects)  
   *Open source tools and projects in the internet archiving space.*
   - [Bookmarking Services](#bookmarking-services)
   - [Well-Known Open Source Projects](#from-the-archiveorg--archive-it-teams)
@@ -27,14 +27,14 @@ Whether you want to learn which organizations are the big players in the web arc
   - [ArchiveBox Alternatives](#other-archivebox-alternatives)
   - [Smaller Utilities](#smaller-utilities)
 
-- [Reading List](#Reading-List)  
+- [Reading List](#reading-list)  
   *Articles, posts, and blogs relevant to ArchiveBox and web archiving in general.*
-  - [Blogs](#Blogs)
-  - [Articles](#Articles)
+  - [Blogs](#blogs-friends-of-archivebox)
+  - [Articles](#articles-we-like-about-internet-archiving)
   - [ArchiveBox-Specific Posts, Tutorials, and Guides](#archivebox-specific-posts-tutorials-and-guides)
   - [ArchiveBox Discussions in News & Social Media](#archivebox-discussions-in-news--social-media)
 
-- [Communities](#Communities)  
+- [Communities](#communities)  
   *A collection of the most active internet archiving communities and initiatives.*
   - [Most Active Web-Archiving Communities](#most-active-communities)
   - [Other Web Archiving Communities](#web-archiving-communities)
@@ -504,5 +504,5 @@ You can find more organizations and initiatives on these other lists:
 [![](https://img.shields.io/badge/Donate-Archive.org-%23115D76.svg)](https://archive.org/donate/)
 
 <br/><br/>
-<small><a href="#contents">^ &nbsp; Back to Top &nbsp; ^</a></small>
+<small><a href="#web-archiving-community">^ &nbsp; Back to Top &nbsp; ^</a></small>
 </div>

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.71",
+  "version": "0.9.72rc28",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

publicsite/ar/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">تكوين ويكي</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">إعدادات Docker</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">تثبيت التبعيات</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">تثبيت التبعيات</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">خيارات تثبيت Chromium</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">ملفات تعريف الارتباط وملامح المتصفح المسجلة الدخول</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">تثبيت استكشاف الأخطاء وإصلاحها</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">قائمة archivebox -- json -- مع رؤوس&amp;gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">قائمة archivebox -- csv = الطابع الزمني ، عنوان url ، العنوان&amp;gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">مستندات تخطيط القرص</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">مستندات تخطيط القرص</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">خلفية تخزين: SMB ، NFS ، S3 ، B2 ، محرك جوجل</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">نشر أرشيفك</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">ترقية أو دمج المحفوظات</a></li>

publicsite/es/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Wiki de configuración</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Configuración de la ventana acoplable</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Instalar dependencias</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Instalar dependencias</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Opciones de instalación de Chrome</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookies y perfiles de navegador registrados</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Solución de problemas de instalación</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">lista de cuadros de archivo --json --con-encabezados &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">lista de cuadros de archivo --csv=marca de tiempo,url,título &gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Documentos de diseño de disco</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Documentos de diseño de disco</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Backends de almacenamiento: SMB, NFS, S3, B2, Google Drive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Publicando su archivo</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Actualizar o fusionar archivos</a></li>

publicsite/fr/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Wiki de configuration</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Configuration du menu Docker</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Installer les dépendances</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Installer les dépendances</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Options d'installation de Chrome</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookies et profils de navigateur connectés</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Dépannage des installations</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">liste des boîtes d'archives --json --with-headers &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">liste des boîtes d'archives --csv=horodatage,url,titre &gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Documentation sur la configuration du disque</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Documentation sur la configuration du disque</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Backends de stockage : SMB, NFS, S3, B2, Google Drive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Publier vos archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Mise à niveau ou fusion d'archives</a></li>

publicsite/index.html

@@ -432,7 +432,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
           <ul class="readme-list">
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration wiki</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker configuration</a></li>
-            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Install dependencies</a></li>
+            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Install dependencies</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Chromium install options</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookies and logged-in browser profiles</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Troubleshooting installs</a></li>
@@ -542,7 +542,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">archivebox list --json --with-headers &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">archivebox list --csv=timestamp,url,title &gt; index.csv</span></pre></div></div>
           <ul class="readme-list">
-            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Disk layout docs</a></li>
+            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Disk layout docs</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Storage backends: SMB, NFS, S3, B2, Google Drive</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Publishing your archive</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Upgrading or merging archives</a></li>

publicsite/ru/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Конфигурация вики</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Конфигурация докера</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Установить зависимости</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Установить зависимости</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Варианты установки хрома</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Файлы cookie и профили браузера для входа в систему</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Устранение неполадок при установке</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">список архивных ящиков --json --with-headers &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">список архивных ящиков --csv=метка времени,url,заголовок &gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Документация по структуре диска</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Документация по структуре диска</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Серверные хранилища: SMB, NFS, S3, B2, Google Drive.</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Публикация вашего архива</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Обновление или объединение архивов</a></li>

publicsite/zh/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">配置维基</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker配置</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">安装依赖项</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">安装依赖项</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">铬安装选项</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookie 和登录的浏览器配置文件</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">安装故障排除</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">archivebox 列表 --json --with-headers &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">archivebox 列表 --csv=时间戳,url,标题 &gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">磁盘布局文档</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">磁盘布局文档</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">存储后端：SMB、NFS、S3、B2、Google Drive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">发布您的档案</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">升级或合并档案</a></li>

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.71"
+version = "0.9.72rc28"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -77,10 +77,10 @@ dependencies = [
     "w3lib>=2.2.1",          # used for parsing content-type encoding from http response headers & html tags
     ### Extractor dependencies (runtime binaries resolved through abxpkg)
     ### Binary/Package Management
-    "abxbus==2.5.70",                 # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abxpkg==1.13.21",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins[opencode]==1.13.109", # optional AI runtime clients are owned by the plugin
-    "abx-dl==1.13.79",       # shared ArchiveBox downloader package
+    "abxbus==2.5.73",                 # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abxpkg==1.13.22",                # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abx-plugins[opencode]==1.13.120", # optional AI runtime clients are owned by the plugin
+    "abx-dl==1.13.87",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
     "browser-cookie3>=0.20.1",
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.71"
+current_version = "v0.9.72rc28"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-23T19:26:02.6709672Z"
+exclude-newer = "2026-09-29T06:21:33.380999932Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-28T19:26:01.670984315Z", span = "PT1S" }
+abxbus = { timestamp = "2026-10-04T06:21:32.381015095Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.13.79"
+version = "1.13.87"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f5/59/e178d330d244351f01026a2c261c7c0d8067c0efcbef9cf095635fa97579/abx_dl-1.13.79.tar.gz", hash = "sha256:8825b06570df48f3ecdd525e8fca09a2d8dac77b8ba059efaac7d0df845b578a", size = 97588, upload-time = "2026-09-28T19:23:47.984Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/d0/28/40d59e443d1661948dfe1bc2fc16ac57a841582ce57f9e614e4b44b05aa9/abx_dl-1.13.87.tar.gz", hash = "sha256:65603f039a539991dc699efa2046e2d4becbccb096105191c5a23cc4a95dfb7c", size = 97710, upload-time = "2026-10-04T06:18:55.925Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/60/6f/f7de1fd42bbcdf56769a7277ca261007525d2a482544fe03d509b3cc6805/abx_dl-1.13.79-py3-none-any.whl", hash = "sha256:3b9b8426da6c0fcd55583d0f26c037561b40982866c4a12f481c8f481fbe56b2", size = 102934, upload-time = "2026-09-28T19:23:49.287Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/d6/cbcdd7b776d395ec2d074b50e6265db29f0e53d7dfe380de06d549f3d30c/abx_dl-1.13.87-py3-none-any.whl", hash = "sha256:a9e31210fce7cc0368d372b1a6d364339e369a675b4e2f888770caf25f4189da", size = 103084, upload-time = "2026-10-04T06:18:54.234Z" },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.13.109"
+version = "1.13.120"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/22/e236ab70ff3c24c3be7f511c202e9943b4acf7988ca180f6593a959cf190/abx_plugins-1.13.109.tar.gz", hash = "sha256:056556249fbe5e1aa5eca3cca84923fab9a4a1fcfd0d3f82c4911de97f2e49f5", size = 461059, upload-time = "2026-09-28T19:08:05.06Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/71/9a/0605e563e66e9ec46231d68ac174ecd4c36d923106ed55b8e1b140cfa918/abx_plugins-1.13.120.tar.gz", hash = "sha256:b17ff0023f59c79f96d4c452528e05f607e460188d0c2b67cc03b2655c5ccb47", size = 461646, upload-time = "2026-10-04T06:06:04.944Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a4/c9/6331394242f8c5afa1fb03abfd805e509add8a7d03c14b9e126e6724af20/abx_plugins-1.13.109-py3-none-any.whl", hash = "sha256:5bdecde68a9bc2265a26b95205c9214a07e39328c981c8700234f577a57d0ba7", size = 673893, upload-time = "2026-09-28T19:08:06.808Z" },
+    { url = "https://files.pythonhosted.org/packages/1f/e8/7f0360b8519c6c956291258fb569a7c1b38c0f5d9a9c8573e0bfd4733520/abx_plugins-1.13.120-py3-none-any.whl", hash = "sha256:43afb24fb06c02fccf3d230326621e6b61bf1ca9faacea434ed3a8245e58931d", size = 674577, upload-time = "2026-10-04T06:06:03.212Z" },
 ]
 
 [package.optional-dependencies]
@@ -68,21 +68,21 @@ opencode = [
 
 [[package]]
 name = "abxbus"
-version = "2.5.70"
+version = "2.5.73"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/52/80/43ab38e5a88a6c8864c2460e0685f946f9fcd6cb4776033611107d70ecd6/abxbus-2.5.70.tar.gz", hash = "sha256:e2a6593d67ed619833a6a2b2a282c58312404f166436a0991f5017d51f45fbe5", size = 134824, upload-time = "2026-09-28T18:36:05.082Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/90/d3/175f38969118aba03326140fdb3d0582ced08026c45cfe4773946d391d0f/abxbus-2.5.73.tar.gz", hash = "sha256:21ed000fdf6dc0e633bac4250d4354e671d05312519aad5078c5bc9e2581dc6a", size = 139209, upload-time = "2026-10-02T18:32:33.762Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/4d/0b/67ebd55f417b641f05b9ba07d738b75b33924026cda220c9d4f771d6e16e/abxbus-2.5.70-py3-none-any.whl", hash = "sha256:a33727a432ec6729d3bd0674d36bb82d87a1faac66480a3eeddaee23e6960a25", size = 128460, upload-time = "2026-09-28T18:36:06.186Z" },
+    { url = "https://files.pythonhosted.org/packages/14/f6/256602ce4b213a13eca3ae2a61659e64881c3f1c9bddd9a4b6d8fb95e37f/abxbus-2.5.73-py3-none-any.whl", hash = "sha256:c20d0bb4dce4ba34ea73ecdae09f118a3a3e72065c7822614f8e7506e0cfb179", size = 132470, upload-time = "2026-10-02T18:32:32.384Z" },
 ]
 
 [[package]]
 name = "abxpkg"
-version = "1.13.21"
+version = "1.13.22"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -90,9 +90,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/0f/bb/56f863aa00e8e9311c0c4af059e7f0ebed2242fd40be9f3b4bbb0e70d646/abxpkg-1.13.21.tar.gz", hash = "sha256:9d5e05a0534836ea637e1c94f4555558c5d6a09368e3ee24e5a056812afc2342", size = 243339, upload-time = "2026-09-28T18:51:45.417Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/30/0c/1f54a00491eeb26b343794951d46cdeab77849c854224d343d1941912f71/abxpkg-1.13.22.tar.gz", hash = "sha256:6bde8929e1df014aa6abfc9256581d0e4587c6d717f5a101d710d03af894f00a", size = 243342, upload-time = "2026-10-02T18:49:33.282Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/37/9c/e60dc8ef5ec6dbe9b39c6bd06165f9d018660cba891295ff625d7cf577b1/abxpkg-1.13.21-py3-none-any.whl", hash = "sha256:b796b99dbbdfdafa3ab05f881ad505539deabbe14eb7b358811f12c47c5a47ad", size = 254735, upload-time = "2026-09-28T18:51:43.415Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/2a/84250b8eeec73530da8afaadcfbc8da87fa340a63d4f7aa18f5553610645/abxpkg-1.13.22-py3-none-any.whl", hash = "sha256:b04b6b79a17d1d876b88fe91ad9c64933f2cc5adf7db44d54ee417f2d71b67a2", size = 254735, upload-time = "2026-10-02T18:49:31.126Z" },
 ]
 
 [[package]]
@@ -127,7 +127,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.71"
+version = "0.9.72rc28"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -227,10 +227,10 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.13.79" },
-    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.109" },
-    { name = "abxbus", specifier = "==2.5.70" },
-    { name = "abxpkg", specifier = "==1.13.21" },
+    { name = "abx-dl", specifier = "==1.13.87" },
+    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.120" },
+    { name = "abxbus", specifier = "==2.5.73" },
+    { name = "abxpkg", specifier = "==1.13.22" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
     { name = "atomicwrites", specifier = "==1.4.1" },
     { name = "base32-crockford", specifier = ">=0.3.0" },