Bump release version to 0.9.72rc45

Resolve relative cookie files before hooks change cwd, reconcile file edits before machine startup saves, and prevent stale cached binaries overriding runtime selections. Cover issue #1896 and real persona localStorage capture through CLI regressions.

.github/CONTRIBUTING.md

@@ -12,7 +12,7 @@
 - https://github.com/ArchiveBox/ArchiveBox/issues
 - https://github.com/ArchiveBox/ArchiveBox/pulls
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 
 ### Development Setup
 

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

.github/lychee.toml

@@ -0,0 +1,38 @@
+# Documentation destinations only: do not crawl archived URLs, app routes, or image assets.
+# Manual run from the repository root with Lychee 0.24.2:
+# git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > /tmp/archivebox-link-inputs.txt
+# lychee --config .github/lychee.toml --root-dir "$PWD" --files-from /tmp/archivebox-link-inputs.txt
+# Explicit tracked inputs include README symlinks and let errors in one file leave others checkable.
+include_fragments = "anchor-only"
+include_verbatim = true
+max_retries = 0
+max_concurrency = 4
+timeout = 20
+no_progress = true
+hidden = true
+extensions = ["md", "html", "py", "rst", "yml", "yaml", "sh", "toml"]
+exclude_path = ['(^|/)(tests|data|apidocs|node_modules|\.git)(/|$)']
+# Includes override this exclude; this is a documentation-link allowlist.
+exclude = ['.*']
+include = [
+ '^https://github\.com/ArchiveBox/ArchiveBox/blob/[^/]+/.*\.md(?:#|$)',
+ '^https://raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/',
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki(?:/|#|$)|/?#)',
+ '^https?://docs\.archivebox\.io(?:/|$)',
+ '^file:.*\.md(?:#|$)',
+ '^file:.*/docs/[^/.#]+(?:#|$)',
+]
+fallback_extensions = ["md"]
+remap = [
+ # Lychee's Markdown slugger drops variation selectors. Check these against rendered HTML.
+
+ '^file:.*/README\.md(#%EF%B8%8F-cli-usage)$ https://github.com/ArchiveBox/ArchiveBox${1}',
+ '^file:.*/docs/Install\.md(#option-a-docker--docker-compose-setup-%EF%B8%8F)$ https://github.com/ArchiveBox/ArchiveBox/wiki/Install${1}',
+ # The wiki accepts this case alias, but raw file paths are case-sensitive.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/Setting-up-Storage$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/Setting-Up-Storage.md',
+
+ # Bare missing wiki pages silently redirect to Home; raw pages return an actual 404.
+ # Keep fragment URLs on the rendered wiki to catch published heading drift.
+ '^https?://github\.com/(?:ArchiveBox|pirate)/ArchiveBox/wiki/([^#/?]+)$ https://raw.githubusercontent.com/wiki/ArchiveBox/ArchiveBox/${1}.md',
+]
+preprocess = { command = "bin/lychee_preprocess.py" }

.github/pages/artifacts.py

@@ -14,7 +14,7 @@ def trusted(run, repo, workflow, branch, successful=True):
     return (
         (run.get("head_repository") or {}).get("full_name", "").lower() == repo.lower()
         and run["head_branch"] == branch
-        and run["event"] in ("push", "workflow_dispatch")
+        and run["event"] in ("push", "workflow_dispatch", "schedule")
         and run["path"] == f".github/workflows/{workflow}"
         and (not successful or run["conclusion"] == "success")
     )

.github/test-durations.json

@@ -2,9 +2,10 @@
   "source_runs": [
     "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36302511923",
     "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552",
-    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36454992009"
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/36454992009",
+    "https://github.com/ArchiveBox/ArchiveBox/actions/runs/37186118498"
   ],
-  "measurement": "Whole job seconds, including setup and excluding queue time. Hosted unbatched files use the latest complete successful run; other files retain their individual baseline measurement. Unknown files stay independent.",
+  "measurement": "Whole job seconds, including setup and excluding queue time. Hosted unbatched files use the latest complete successful run; other files retain their individual baseline measurement. test_issue_1896 uses its successful ugNAS job measurement from run 37186118498. Unknown files stay independent.",
   "seconds": {
     "archivebox/tests/test_api_v1_auth_browser_session.py": 29,
     "archivebox/tests/test_api_v1_auth_check_api_token.py": 27,
@@ -99,6 +100,7 @@
     "archivebox/tests/test_frozen_crawl_config.py": 47,
     "archivebox/tests/test_hook_result_lifecycle.py": 78,
     "archivebox/tests/test_hooks.py": 90,
+    "archivebox/tests/test_issue_1896.py": 485,
     "archivebox/tests/test_machine_models.py": 52,
     "archivebox/tests/test_machine_service.py": 89,
     "archivebox/tests/test_migrations_04_to_09.py": 36,

.github/workflows/ci.yml

@@ -45,13 +45,6 @@ jobs:
     uses: ./.github/workflows/test-parallel.yml
     secrets: inherit
 
-  documentation:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
-    name: Documentation and root tests
-    uses: ./.github/workflows/docs.yml
-    secrets: inherit
-
   codeql:
     needs: prepare
     if: needs.prepare.outputs.run_jobs == 'true'
@@ -83,7 +76,6 @@ jobs:
       - lint
       - install-cli-platform
       - tests
-      - documentation
       - codeql
       - python-artifacts
       - docker-digests
@@ -94,7 +86,6 @@ jobs:
           LINT_RESULT: ${{ needs.lint.result }}
           INSTALL_CLI_PLATFORM_RESULT: ${{ needs.install-cli-platform.result }}
           TESTS_RESULT: ${{ needs.tests.result }}
-          DOCUMENTATION_RESULT: ${{ needs.documentation.result }}
           CODEQL_RESULT: ${{ needs.codeql.result }}
           PYTHON_ARTIFACTS_RESULT: ${{ needs.python-artifacts.result }}
           DOCKER_DIGESTS_RESULT: ${{ needs.docker-digests.result }}
@@ -103,7 +94,6 @@ jobs:
             "$LINT_RESULT" \
             "$INSTALL_CLI_PLATFORM_RESULT" \
             "$TESTS_RESULT" \
-            "$DOCUMENTATION_RESULT" \
             "$CODEQL_RESULT" \
             "$PYTHON_ARTIFACTS_RESULT" \
             "$DOCKER_DIGESTS_RESULT"

.github/workflows/docker.yml

@@ -350,6 +350,7 @@ jobs:
           (( COMPRESSED_SIZE < LIMIT ))
 
       - name: Resolve real Docker CLI test dependencies through abxpkg
+        id: mount_dependencies
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash
         run: |
@@ -371,6 +372,15 @@ jobs:
             test -x "$binary"
           done
 
+      - name: Diagnose failed mount dependency installation
+        if: failure() && steps.mount_dependencies.conclusion == 'failure'
+        shell: bash
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          sudo journalctl --since '-10 minutes' --no-pager \
+            -u smbd -u nmbd -u samba-ad-dc -u nfs-server
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
       - name: Validate Docker UID and GID behavior on real NFS and SMB mounts
         if: inputs.full_tests && matrix.platform == 'linux/amd64'
         shell: bash

.github/workflows/docs-links.yml

@@ -0,0 +1,38 @@
+name: Documentation link checks
+
+on:
+  schedule:
+    - cron: '17 10 * * *' # Daily at 10:17 UTC, away from the busy top of the hour.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PYTHONIOENCODING: utf-8
+
+jobs:
+  links:
+    name: Documentation links and anchors
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
+        with:
+          version: "0.11.3"
+          enable-cache: false
+      - name: List tracked documentation and source inputs
+        run: |
+          git ls-files -- '*.md' '*.html' '*.py' '*.rst' '*.yml' '*.yaml' '*.sh' '*.toml' > "$RUNNER_TEMP/link-inputs.txt"
+      - uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2
+        with:
+          lycheeVersion: v0.24.2
+          args: >-
+            --config .github/lychee.toml
+            --root-dir "${{ github.workspace }}"
+            --files-from "${{ runner.temp }}/link-inputs.txt"
+          fail: true
+          failIfEmpty: true

.github/workflows/docs.yml

@@ -1,7 +1,17 @@
 name: Documentation tests
 
 on:
-  workflow_call:
+  schedule:
+    - cron: '7 5,17 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 
 env:
   PYTHONIOENCODING: utf-8
@@ -9,7 +19,18 @@ env:
   SHOW_PROGRESS: "False"
 
 jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
   discover:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     name: Discover documentation examples
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -158,3 +179,17 @@ jobs:
           "$SUDO_BINARY" \
             --preserve-env=PATH,ABXPKG_LIB_DIR,BASH_BINARY,PYTHON_BINARY,SUDO_BINARY,UV_BINARY \
             "$UV_BINARY" run --no-cache --no-sync python docs/test_codeblocks_manifest.py run-group '${{ matrix.snippet_ids }}'
+      - name: Diagnose failed Linux package installation
+        if: failure() && runner.os == 'Linux'
+        run: |
+          sudo tail -n 120 /var/log/apt/term.log /var/log/dpkg.log
+          apt-config dump Acquire::Retries Acquire::http::Timeout Acquire::https::Timeout
+          ps -eo pid,ppid,stat,etimes,wchan:32,comm
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, discover, examples]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/macos.yml

@@ -0,0 +1,48 @@
+name: macOS compatibility
+
+on:
+  schedule:
+    - cron: '51 2,10,18 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
+
+  install-cli:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/test.yml
+    with:
+      platform: macos
+    secrets: inherit
+
+  python-artifacts:
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
+    uses: ./.github/workflows/pip.yml
+    with:
+      platforms: '["macos-15"]'
+    secrets: inherit
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, install-cli, python-artifacts]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled macOS coverage completed successfully.'

.github/workflows/pip.yml

@@ -3,6 +3,11 @@ name: Build Pip package
 on:
   workflow_call:
     inputs:
+      platforms:
+        description: Runner matrix for package compatibility
+        required: false
+        default: '["ubuntu-24.04"]'
+        type: string
       full_tests:
         description: Run the full package compatibility matrix instead of one release smoke check
         required: false
@@ -95,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-24.04, macos-15]
+        os: ${{ fromJSON(inputs.platforms) }}
         python: ["3.13", "3.14"]
         artifact: [wheel, sdist]
 

.github/workflows/screenshots.yml

@@ -1,38 +1,31 @@
 name: Capture publicsite screenshots
 
 on:
-  push:
-    branches:
-      - dev
-    paths:
-      - archivebox/**
-      - bin/collect_ui_screenshots.*
-      - bin/generate_ui_screenshot_gallery.py
-      - bin/take_screenshot.js
-      - docs/Screenshots.md
-      - pyproject.toml
-      - uv.lock
-      - .github/workflows/screenshots.yml
-      - .github/workflows/prepare-release.yml
+  schedule:
+    - cron: '37 5,17 * * *'
   workflow_dispatch:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
-  group: ${{ github.event_name == 'workflow_dispatch' && format('publicsite-captures-manual-{0}', github.run_id) || 'publicsite-captures' }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
 jobs:
-  prepare:
-    permissions:
-      contents: write
-    uses: ./.github/workflows/prepare-release.yml
-    secrets: inherit
+  changes:
+    runs-on: ubuntu-24.04
+    outputs:
+      changed: ${{ steps.check.outputs.changed }}
+      key: ${{ steps.check.outputs.key }}
+    steps:
+      - id: check
+        uses: ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main
 
   deploy:
-    needs: prepare
-    if: needs.prepare.outputs.run_jobs == 'true'
+    needs: changes
+    if: needs.changes.outputs.changed == 'true'
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout
@@ -78,3 +71,11 @@ jobs:
           path: publicsite/screenshots
           if-no-files-found: error
           retention-days: 90
+
+  complete:
+    name: 'Scheduled inputs: ${{ needs.changes.outputs.key }}'
+    needs: [changes, deploy]
+    if: needs.changes.outputs.changed == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - run: echo 'Scheduled work completed successfully.'

.github/workflows/test-parallel.yml

@@ -36,16 +36,9 @@ jobs:
       - name: Select available CI capacity
         id: runner
         if: vars.UGNAS_CI_ENABLED == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main')
-        uses: mikehardy/runner-fallback-action@dc7732f9e532c451b2527b288ba8d58fd4b0a4ca # v1.1.0
+        uses: ArchiveBox/monorepo/.github/actions/select-ci-runner@main
         with:
           github-token: ${{ secrets.RUNNER_STATUS_TOKEN }}
-          organization: ArchiveBox
-          primary-runner: self-hosted,ugnas-linux
-          fallback-runner: ubuntu-24.04
-          # Jobs queued for this pool must not be mistaken for idle runner slots:
-          # requiring MAX_JOBS idle runners prevents assigning a second wave.
-          primaries-required: ${{ vars.UGNAS_CI_MIN_IDLE || 1 }}
-          fallback-on-error: true
 
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -106,7 +99,7 @@ jobs:
     needs: discover-tests
     # Opt in trusted branches only; PRs always use GitHub-hosted runners.
     # Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
-    # The hosted discovery job falls back when workers are offline/busy or the
+    # The hosted discovery job falls back when workers are offline or the
     # availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
     runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
     env:
@@ -143,6 +136,30 @@ jobs:
           enable-cache: false
           cache-dependency-glob: uv.lock
 
+      # Restored binaries still pass normal abxpkg resolution/version checks.
+      # Installation and isolation tests explicitly choose fresh temporary libs.
+      - name: Identify runtime dependency cache
+        shell: bash
+        run: |
+          uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
+          import hashlib
+          import json
+          import tomllib
+          from pathlib import Path
+
+          packages = tomllib.loads(Path("uv.lock").read_text())["package"]
+          versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
+          print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
+          PYTHON
+
+      - name: Reuse validated runtime dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+        with:
+          path: ${{ runner.temp }}/archivebox-lib
+          key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ env.CI_PYTHON_VERSION }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}
+          restore-keys: |
+            abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ env.CI_PYTHON_VERSION }}-${{ matrix.test.name }}-
+
       - name: Resolve optional build dependencies through abxpkg
         if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
         env:
@@ -196,11 +213,8 @@ jobs:
             export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
             export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
             export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
-            if (( ${#test_paths[@]} > 1 )); then
-              export ABXPKG_LIB_DIR="$RUNNER_TEMP/archivebox-lib/$test_slug"
-            fi
             mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
-            uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" --durations=20 \
+            uv run --no-cache --no-sync --no-sources pytest -vs -n 2 "$GITHUB_WORKSPACE/$test_path" --durations=20 \
               --basetemp="$RUNNER_TEMP/pytest/$test_slug" \
               --junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
           done

.github/workflows/test.yml

@@ -2,6 +2,12 @@ name: Install and CLI Compatibility
 
 on:
   workflow_call:
+    inputs:
+      platform:
+        description: Platform for install and CLI compatibility
+        required: false
+        default: linux
+        type: string
 
 env:
   DOCKER_IMAGE: archivebox-ci
@@ -19,16 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-24.04
-            os_name: linux
-            python: "3.14"
-          - os: macos-15
-            os_name: macOS
-            python: "3.13"
-          - os: macos-15
-            os_name: macOS
-            python: "3.14"
+        include: ${{ fromJSON(inputs.platform == 'macos' && '[{"os":"macos-15","os_name":"macOS","python":"3.13"},{"os":"macos-15","os_name":"macOS","python":"3.14"}]' || '[{"os":"ubuntu-24.04","os_name":"linux","python":"3.14"}]') }}
 
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

README.md

@@ -123,13 +123,13 @@ curl -fsSL 'https://get.archivebox.io' | bash
 ## Key Features
 
 - [**Free & open source**](https://github.com/ArchiveBox/ArchiveBox/blob/dev/LICENSE), own your own data & maintain your privacy by self-hosting
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) with [modular dependencies](#dependencies) and [support for Google Drive/NFS/SMB/S3/B2/etc.](https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage)
 - [**Comprehensive documentation**](https://github.com/ArchiveBox/ArchiveBox/wiki), [active development](https://github.com/ArchiveBox/ArchiveBox/wiki/Roadmap), and [rich community](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community)
 - [**Extracts a wide variety of content out-of-the-box**](https://github.com/ArchiveBox/ArchiveBox/issues/51): [media (yt-dlp), articles (readability), code (git), etc.](#output-formats)
 - [**Supports scheduled/realtime importing**](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving) from [many types of sources](#input-formats)
 - [**Uses standard, durable, long-term formats**](#output-formats) like HTML, JSON, PDF, PNG, MP4, TXT, and WARC
-- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#UI-Usage), [Python API](https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
-- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#save_archive_dot_org) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#stealth-mode) for local-only mode)
+- [**Powerful CLI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage), [**self-hosted web UI**](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#ui-usage), [Python API](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html) (BETA), [REST API](https://github.com/ArchiveBox/ArchiveBox/issues/496) (ALPHA), or [desktop app](https://electron.archivebox.io/)
+- [**Saves all pages to archive.org as well**](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) by default for redundancy (can be [disabled](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archivedotorg_enabled) for local-only mode)
 - Support for archiving [content requiring login/paywall/cookies](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#chrome_user_data_dir) (see wiki security caveats!)
 - Support for running [JS during archiving](https://github.com/ArchiveBox/ArchiveBox/issues/51) to adblock, [infiniscroll](https://github.com/ArchiveBox/ArchiveBox/issues/80), [modal-hide](https://github.com/ArchiveBox/ArchiveBox/issues/175), [thread-expand](https://github.com/ArchiveBox/ArchiveBox/issues/345)
 
@@ -220,7 +220,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for more usage examples using the C
 <ol>
 <li>Run the shortcut for the <code>uv</code> install method. It installs <code>uv</code> first when needed.
 <pre lang="bash"><code style="white-space: pre-line">curl -fsSL 'https://get.archivebox.io' | bash</code></pre>
-<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-automatic-setup-script">Install: Bare Metal</a> in the Wiki. ➡️</i>
+<i>For more info, see <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-b-uv-install-shortcut">Install: Bare Metal</a> in the Wiki. ➡️</i>
 </li>
 </ol>
 
@@ -431,7 +431,7 @@ For more discussion on third-party hosting options see here: <a href="https://gi
 #### ➡️&nbsp; Next Steps
 
 - Import URLs from some of the supported [Input Formats](#input-formats) or view the supported [Output Formats](#output-formats)...
-- (Optional) [Import browser cookies and settings into a persona](docs/Chromium-Install.md#import-an-existing-browser-profile).
+- (Optional) [Import browser cookies and settings into a persona](https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile).
 - Tweak your UI or archiving behavior [Configuration](#configuration), read about some of the [Caveats](#caveats), or [Troubleshoot](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting)
 - Read about the [Dependencies](#dependencies) used for archiving, the [Upgrading Process](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives), or the [Archive Layout](#archive-layout) on disk...
 - Or check out our full [Documentation](#documentation) or [Community Wiki](#internet-archiving-ecosystem)...
@@ -613,7 +613,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
 <sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
 <br/><br/>
 <a href="https://demo.archivebox.io">DEMO: <code>https://demo.archivebox.io</code></a><br/>
-<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#Caveats">Caveats</a>
+<a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration</a> | <a href="#caveats">Caveats</a>
 <br/>
 </div>
 
@@ -641,7 +641,7 @@ docker run -it -v $PWD:/data archivebox/archivebox:dev add --depth=1 'https://ex
   <i>Save directly from the Share menu or Siri/Shortcuts to your server.</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/64078483-21d7-4eb1-aa6e-9ad55afe45b8" height="22px"/> From manual imports of URLs from RSS, JSON, CSV, TXT, SQL, HTML, Markdown, etc. files  
-  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Import-a-list-of-URLs-from-a-text-file).</i>
+  <i>ArchiveBox supports injesting URLs in [any text-based format](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#import-a-list-of-urls-from-a-text-file).</i>
 
 - <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/32b494e6-4de1-4984-8d88-dc02f18e5c34" height="22px"/> From manually exported [browser history](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) or [browser bookmarks](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive) (in Netscape format)  
   <i>Instructions: <a href="https://support.google.com/chrome/answer/96816?hl=en">Chrome</a>, <a href="https://support.mozilla.org/en-US/kb/export-firefox-bookmarks-to-backup-or-transfer">Firefox</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/assets/511499/24ad068e-0fa6-41f4-a7ff-4c26fc91f71a">Safari</a>, <a href="https://support.microsoft.com/en-us/help/211089/how-to-import-and-export-the-internet-explorer-favorites-folder-to-a-32-bit-version-of-windows">IE</a>, <a href="https://help.opera.com/en/latest/features/#bookmarks:~:text=Click%20the%20import/-,export%20button,-on%20the%20bottom">Opera</a>, <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive">and more...</a></i>
@@ -670,7 +670,7 @@ echo 'any text with <a href="https://example.com">urls</a> in it' | archivebox a
 # echo 'https://example.com' | docker compose run --rm -T archivebox add
 ```
 
-See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#CLI-Usage) page for documentation and examples.
+See the [Usage: CLI](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#cli-usage) page for documentation and examples.
 
 It also includes a built-in scheduled import feature with `archivebox schedule`, handled by the same orchestrator that powers `archivebox server`, so you can pull in URLs from RSS feeds and websites regularly without a separate cron container.
 
@@ -811,7 +811,7 @@ Installing directly on <strong>Windows without Docker or WSL/WSL2/Cygwin is not
 
 <h4>Learn More</h4>
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Wiki: Install (Dependencies)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Wiki: Install (Dependencies)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Wiki: Chromium Install</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Wiki: Upgrading or Merging Archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Wiki: Troubleshooting (Installing)</a></li>
@@ -865,7 +865,7 @@ Each snapshot subfolder includes static metadata and plain extractor output file
 <h4>Learn More</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Wiki: Setting Up Storage (SMB, NFS, S3, B2, Google Drive, etc.)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
@@ -951,7 +951,7 @@ archivebox manage createsuperuser
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview">Wiki: Security Overview</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Wiki: Chromium Install (Setting Up a User Profile)</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">Wiki: Chrome profiles and cookies</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#default_persona">Wiki: Configuration (<code>DEFAULT_PERSONA</code>)</a></li>
 </ul>
 
@@ -1004,7 +1004,7 @@ For various reasons, many large sites (Reddit, Twitter, Cloudflare, etc.) active
 
 
 <ul>
-<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
+<li>Set up a logged-in browser session for archiving by <a href="https://github.com/ArchiveBox/ArchiveBox/blob/dev/docs/Chromium-Install.md#import-an-existing-browser-profile">importing a chrome profile into a persona</a></li>
 <li>Rewrite your URLs before archiving to swap in alternative frontends that are more bot-friendly e.g.<br>
 <code>reddit.com/some/url</code> -&gt; <code>teddit.net/some/url</code>: <a href="https://github.com/mendel5/alternative-front-ends">https://github.com/mendel5/alternative-front-ends</a></li>
 </ul>
@@ -1065,7 +1065,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
 <h4>Learn More</h4>
 
 <ul>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output-Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user">Wiki: Security Overview (Privilege Dropping)</a></li>
@@ -1190,7 +1190,7 @@ By having users store their content locally or within their organizations, we ca
 
 <h3>Comparison With Other Self-Hosted Archiving Options</h3>
 
-ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
+ArchiveBox differentiates itself from [similar self-hosted projects](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects) by providing both a comprehensive CLI interface for managing your archive, a Web UI that can be used either independently or together with the CLI, and a simple on-disk data format that can be used without either.
 
 
 *If you want better fidelity for very complex interactive pages with heavy JS/streams/API requests, check out [ArchiveWeb.page](https://archiveweb.page) and [ReplayWeb.page](https://replayweb.page).*
@@ -1199,7 +1199,7 @@ ArchiveBox differentiates itself from [similar self-hosted projects](https://git
 
 *If you need more advanced recursive spider/crawling ability beyond `--depth=1`, check out [Browsertrix](https://github.com/webrecorder/browsertrix-crawler), [Photon](https://github.com/s0md3v/Photon), or [Scrapy](https://scrapy.org/) and pipe the outputted URLs into ArchiveBox.*
 
-For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#Web-Archiving-Projects)...
+For more alternatives, see our [list here](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community#web-archiving-projects)...
 
 ArchiveBox is neither the highest fidelity nor the simplest tool available for self-hosted archiving, rather it's a jack-of-all-trades that tries to do most things well by default. We encourage you to try these other tools made by our friends if ArchiveBox isn't suited to your needs.
 
@@ -1254,7 +1254,7 @@ We use the [ArchiveBox GitHub Wiki](https://github.com/ArchiveBox/ArchiveBox/wik
 - [Usage](https://github.com/ArchiveBox/ArchiveBox/wiki/Usage)
 - [Configuration](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
 - [Supported Sources](https://github.com/ArchiveBox/ArchiveBox/wiki/Quickstart#2-get-your-list-of-urls-to-archive)
-- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox/wiki#can-save-these-things-for-each-site)
+- [Supported Outputs](https://github.com/ArchiveBox/ArchiveBox#output-formats)
 - [Scheduled Archiving](https://github.com/ArchiveBox/ArchiveBox/wiki/Scheduled-Archiving)
 
 ## Advanced

archivebox/api/v1_api.py

@@ -2,6 +2,8 @@ __package__ = "archivebox.api"
 
 
 from io import StringIO
+from functools import lru_cache
+from typing import Any
 from traceback import format_exception
 from contextlib import redirect_stdout, redirect_stderr
 
@@ -10,6 +12,7 @@ from django.core.exceptions import ObjectDoesNotExist, EmptyResultSet, Permissio
 from django.contrib.auth.models import User
 
 from ninja import NinjaAPI, Swagger
+from ninja.openapi.schema import OpenAPISchema
 
 # TODO: explore adding https://eadwincode.github.io/django-ninja-extra/
 
@@ -49,6 +52,16 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
 
 
 class NinjaAPIWithIOCapture(NinjaAPI):
+    def get_openapi_schema(self, *, path_prefix: str | None = None, path_params: dict[str, Any] | None = None) -> OpenAPISchema:
+        prefix = path_prefix if path_prefix is not None else self.get_root_path(path_params or {})
+        return self._schema_for_prefix(prefix)
+
+    @lru_cache(maxsize=8)
+    def _schema_for_prefix(self, path_prefix: str) -> OpenAPISchema:
+        # Routes are registered at startup. Server discovery can reuse their
+        # schema while preserving distinct URL prefixes for mounted instances.
+        return super().get_openapi_schema(path_prefix=path_prefix)
+
     def create_temporal_response(self, request: HttpRequest) -> HttpResponse:
         stdout, stderr = StringIO(), StringIO()
 

archivebox/api/v1_core.py

@@ -392,13 +392,13 @@ def _write_archiveresult_files(
     mime_types = _get_archiveresult_upload_form_values(request, "mime_types", "mime_type")
     chunk_output_path = _get_archiveresult_upload_form_value(request, "chunk_output_path")
 
+    output_files = dict(existing_output_files or {})
+    if not files:
+        return output_files
+
     snapshot_dir = snapshot.output_dir
     plugin_dir = snapshot_dir / plugin_name
     storage = FileSystemStorage(location=str(plugin_dir))
-    output_files = dict(existing_output_files or {})
-
-    if not files:
-        return output_files
 
     if chunk_output_path:
         if len(files) != 1:
@@ -630,6 +630,8 @@ def patch_archiveresult(
 class SnapshotSchema(Schema):
     TYPE: str = "core.models.Snapshot"
     id: UUID
+    crawl_id: UUID
+    persona: str | None
     created_by_id: str
     created_by_username: str
     created_at: datetime
@@ -648,6 +650,17 @@ class SnapshotSchema(Schema):
     num_archiveresults: int
     archiveresults: list[MinimalArchiveResultSchema]
 
+    @staticmethod
+    def resolve_archive_path(obj):
+        return obj.archive_path_from_db
+
+    @staticmethod
+    def resolve_persona(obj):
+        # A deduplicated submission can reuse a capture from a different
+        # persona. Report its actual DB-backed owner, not the new request's choice.
+        persona = obj.crawl.resolve_persona()
+        return persona.name if persona else None
+
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by.pk)
@@ -700,6 +713,7 @@ class SnapshotDeleteResponseSchema(Schema):
     snapshot_id: str
     crawl_id: str
     deleted_count: int
+    queued_count: int
 
 
 def normalize_tag_list(tags: list[str] | None = None) -> list[str]:
@@ -938,6 +952,15 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
     # lock. The unique insert recovery and CAS update below are the request-side
     # coordination boundary for this idempotent metadata sync.
     snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None and bool(crawl.get_current_config().get("ONLY_NEW", True)):
+        # Uploading browser metadata is part of the same submission as /add.
+        # It must honor the runner's deduplication policy, even if it arrives
+        # before the runner has processed that crawl. Report the actual owner
+        # so clients never mistake a reused capture for one they just created.
+        snapshot = Snapshot.objects.filter(url=data.url).exclude(status=Snapshot.DELETING_STATE).order_by("-created_at").first()
+        if snapshot is not None:
+            setattr(request, "with_archiveresults", False)
+            return snapshot
     if snapshot is None:
         try:
             snapshot = Snapshot.objects.create(
@@ -971,11 +994,6 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    try:
-        snapshot.ensure_crawl_symlink()
-    except Exception:
-        pass
-
     setattr(request, "with_archiveresults", False)
     return snapshot
 
@@ -1037,22 +1055,23 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
 
 @router.delete("/snapshot/{snapshot_id}", response=SnapshotDeleteResponseSchema, url_name="delete_snapshot")
 def delete_snapshot(request: HttpRequest, snapshot_id: str):
+    """Queue deletion; the normal server runner stops writers and removes files.
+
+    The snapshot remains readable with status=deleting until cleanup succeeds.
+    queued_count reports acceptance; deleted_count is zero in this response.
+    """
     snapshot = get_snapshot(request, snapshot_id, with_archiveresults=False)
     snapshot_id_str = str(snapshot.id)
-    crawl_id_str = str(snapshot.crawl.pk)
-    snapshot.cancel()
-
-    from archivebox.services.runner import run_pending_crawls
-
-    with crawl_lifecycle_lock(crawl_id_str):
-        run_pending_crawls(crawl_id=crawl_id_str, daemon=False)
-        snapshot = get_snapshot(request, snapshot_id_str, with_archiveresults=False)
-        deleted_count, _ = snapshot.delete()
+    crawl_id_str = str(snapshot.crawl_id)
+    # Use the same durable queue as the admin delete action. Stopping writers
+    # and removing archive files belongs to the runner, never the HTTP request.
+    queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
     return {
         "success": True,
         "snapshot_id": snapshot_id_str,
         "crawl_id": crawl_id_str,
-        "deleted_count": deleted_count,
+        "deleted_count": 0,
+        "queued_count": queued_count,
     }
 
 

archivebox/api/v1_personas.py

@@ -5,9 +5,11 @@ from datetime import datetime
 from typing import Any, Literal
 from uuid import UUID
 
+from django.core.exceptions import ValidationError
 from django.db.models import Q
 from django.http import HttpRequest
 from ninja import Router, Schema
+from ninja.errors import HttpError
 from ninja.pagination import paginate
 from pydantic import Field
 
@@ -108,10 +110,9 @@ def browser_settings_to_config(extension_persona_id: str, settings: PersonaBrows
 
 
 def find_persona(extension_persona_id: str, name: str) -> Persona | None:
+    named = Persona.find_named(name)
     return (
-        Persona.objects.filter(
-            Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(name=name),
-        )
+        Persona.objects.filter(Q(config__BROWSER_EXTENSION_PERSONA_ID=extension_persona_id) | Q(pk=named.pk if named else None))
         .order_by("created_at")
         .first()
     )
@@ -138,7 +139,11 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     if not is_valid:
         raise ValueError(error_message)
 
-    persona = find_persona(payload.extension_persona_id, name)
+    try:
+        persona = find_persona(payload.extension_persona_id, name)
+        (persona or Persona(name=name)).validate_name(persona.name if persona else name)
+    except ValidationError as err:
+        raise HttpError(409, "; ".join(err.messages)) from err
     created = persona is None
     if persona is None:
         persona = Persona(name=name)

archivebox/base_models/models.py

@@ -9,6 +9,7 @@ from typing import Any
 from archivebox.uuid_compat import CompactUUIDField, uuid7
 from pathlib import Path
 
+from django.core.exceptions import FieldDoesNotExist
 from django.db import models
 from django.db.models import F
 from django.db import transaction
@@ -23,6 +24,103 @@ from django_stubs_ext.db.models import TypedModelMeta
 from archivebox.config import CONSTANTS
 
 
+class PagedQuerySet(models.QuerySet):
+    def paged_iterator(self, chunk_size: int = 500):
+        """
+        Iterate rows using bounded keyset pages instead of one streaming cursor.
+
+        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
+        cursor open until the full queryset is exhausted. That is fine for
+        read-only exports, but update/migration code does filesystem work and
+        writes while iterating; a long-lived read cursor there can stretch lock
+        waits across thousands of rows. This respects the queryset's existing
+        filters, order_by(), select_related(), and prefetch_related() state; if
+        no ordering is defined, it falls back to primary-key order.
+        """
+        pk_field = self.model._meta.pk.name
+        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
+
+        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        ordering = []
+        for term in raw_ordering:
+            descending = term.startswith("-")
+            field_name = term[1:] if descending else term
+            if field_name == "pk":
+                field_name = pk_field
+            ordering.append(f"-{field_name}" if descending else field_name)
+
+        ordered_field_names = [term.removeprefix("-") for term in ordering]
+        try:
+            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
+                offset = 0
+                while True:
+                    batch = list(self[offset : offset + chunk_size])
+                    if not batch:
+                        break
+                    yield from batch
+                    offset += chunk_size
+                return
+        except (AttributeError, FieldDoesNotExist):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
+        if not any(field_name in unique_field_names for field_name in ordered_field_names):
+            offset = 0
+            while True:
+                batch = list(self[offset : offset + chunk_size])
+                if not batch:
+                    break
+                yield from batch
+                offset += chunk_size
+            return
+
+        last_values = None
+        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
+        while True:
+            batch_qs = self.order_by(*ordering)
+            if last_values is not None:
+                page_filter = models.Q()
+                for idx, term in enumerate(ordering):
+                    descending = term.startswith("-")
+                    field_name = term[1:] if descending else term
+                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
+                    comparison = "lt" if descending else "gt"
+                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
+                batch_qs = batch_qs.filter(page_filter)
+
+            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
+            if not batch_rows:
+                break
+
+            pk_idx = value_field_names.index(pk_field)
+            row_ids = [row[pk_idx] for row in batch_rows]
+            rows_by_id = {obj.pk: obj for obj in self.filter(pk__in=row_ids).order_by()}
+
+            for row in batch_rows:
+                row_id = row[pk_idx]
+                obj = rows_by_id.get(row_id)
+                if obj is not None:
+                    yield obj
+
+            last_values = batch_rows[-1][: len(ordered_field_names)]
+
+
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
         return config
@@ -231,12 +329,9 @@ class ModelWithOutputDir(ModelWithUUID):
 
     _delete_signal_registered = False
 
-    def save(self, *args, **kwargs):
-        super().save(*args, **kwargs)
-        output_dir = Path(self.output_dir)
-        # Avoid holding SQLite write transactions open across slow filesystem work.
-        transaction.on_commit(lambda: output_dir.mkdir(parents=True, exist_ok=True))
-        # Note: index.json is deprecated, models should use write_index_jsonl() for full data
+    # Saving metadata must not resolve or create output paths. The runner and
+    # explicit artifact writers create their own directories when writing files;
+    # on_commit would still make HTTP requests wait on remote archive storage.
 
     @property
     def output_dir_parent(self) -> str:
@@ -282,7 +377,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +396,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/__init__.py

@@ -2,6 +2,7 @@ __package__ = "archivebox.cli"
 __command__ = "archivebox"
 import os
 import sys
+from contextlib import redirect_stdout
 from importlib import import_module
 
 import rich_click as click
@@ -219,6 +220,12 @@ def main(args=None, prog_name=None):
         raise SystemExit(130) from None
     finally:
         sys.unraisablehook = previous_unraisablehook
+        supervisor_util = sys.modules.get("archivebox.workers.supervisord_util")
+        if supervisor_util is not None:
+            # Short commands can start a foreground-owned search daemon too.
+            # Finish its cleanup before another CLI can borrow a dying stack.
+            with redirect_stdout(sys.stderr):
+                supervisor_util.stop_own_supervisord_process()
 
 
 if __name__ == "__main__":

archivebox/cli/archivebox_add.py

@@ -144,7 +144,6 @@ def add(
     persona_name = (persona or "Default").strip() or "Default"
     plugins = plugins or ""
     persona_obj = Persona.get_or_create_named(persona_name)
-    persona_obj.ensure_dirs()
     effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
@@ -217,9 +216,6 @@ def add(
         print(
             "[yellow]\\[*] URLs queued. The background runner will process them (run `archivebox server` or `archivebox run --daemon` if not already running).[/yellow]",
         )
-        from archivebox.services.runner import ensure_background_runner
-
-        ensure_background_runner()
     else:
         # Foreground mode: run full crawl runner until all work is done
         print("[green]\\[*] Starting crawl runner to process crawl...[/green]")

archivebox/cli/archivebox_init.py

@@ -116,6 +116,11 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     # Registration requires the final schema and reconciles legacy file config.
     Machine.current()
 
+    from archivebox.personas.models import Persona
+
+    # Initialize here, not when /add/ is rendered: a GET must not create profiles.
+    Persona.get_or_create_default().ensure_dirs()
+
     from archivebox.misc.db import database_display_location, is_postgres
 
     assert database_exists()

archivebox/cli/archivebox_install.py

@@ -58,7 +58,6 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
-    from archivebox.services.runner import run_due_binary
 
     machine = Machine.current()
     for requested_name in binary_names:
@@ -80,7 +79,13 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
             binary.status = Binary.StatusChoices.QUEUED
             binary.retry_at = timezone.now()
             binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
-        run_due_binary(binary, lock_seconds=60)
+        if binary.status == Binary.StatusChoices.INSTALLED:
+            continue
+        # The synchronous CLI needs the lifecycle's verified installation result,
+        # not the scheduler's "claimed work" result. Reuse its failure/requeue and
+        # health-stat handling so a failed install cannot exit successfully.
+        if not binary.install_claimed(lock_seconds=60):
+            raise click.ClickException(f"Binary {binary_name} is already being installed")
 
 
 def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:

archivebox/cli/archivebox_persona.py

@@ -163,8 +163,9 @@ def create_personas(
             rprint(f'[red]Invalid persona name "{name}": {error_msg}[/red]', file=sys.stderr)
             continue
 
-        persona, created = Persona.objects.get_or_create(
-            name=name,
+        created = Persona.find_named(name) is None
+        persona = Persona.get_or_create_named(
+            name,
             defaults={"config": {"PERMISSIONS": permissions}} if permissions else {},
         )
 
@@ -313,7 +314,9 @@ def update_personas(name: str | None = None) -> int:
             if persona_id:
                 persona = Persona.objects.get(id=persona_id)
             else:
-                persona = Persona.objects.get(name=old_name)
+                persona = Persona.find_named(old_name)
+                if persona is None:
+                    raise Persona.DoesNotExist
 
             # Apply updates from CLI flags
             if name:
@@ -325,7 +328,7 @@ def update_personas(name: str | None = None) -> int:
 
                 # Rename the persona directory too
                 old_path = persona.path
-                persona.name = name
+                persona.name = persona.validate_name(name)
                 new_path = persona.path
 
                 if old_path.exists() and old_path != new_path:

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_server.py

@@ -260,6 +260,12 @@ def server(
     bind_spec = next((arg for arg in runserver_args if arg), "")
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
+    from archivebox.personas.models import Persona
+
+    # Older collections deferred this until /add/. Reconcile before serving any
+    # requests, and report ambiguous profile names before Chrome can use them.
+    Persona.get_or_create_default().ensure_dirs()
+
     if not User.objects.filter(is_superuser=True).exclude(username="system").exists():
         from archivebox.core.routes_util import build_admin_url
 

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))
@@ -182,6 +195,10 @@ def reconcile_known_snapshots(snapshots, wait_for_turn=None):
             snapshot.migrate_filesystem_to_current_version()
         directory = Path(snapshot.output_dir)
         if not directory.is_dir():
+            # Canceling queued metadata can seal a row before the runner ever
+            # creates storage. There is nothing on disk to reconcile for it.
+            if snapshot.downloaded_at is None and not snapshot.archiveresult_set.exists():
+                continue
             raise ValueError(f"Missing snapshot directory: {directory}")
         _reconcile_directory(directory)
         for result in snapshot.archiveresult_set.all():

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -468,6 +487,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -498,7 +522,7 @@ class ArchivingConfig(BaseConfigSet):
             rprint("    (Setting it to somewhere between 30 and 3000 seconds is recommended)", file=sys.stderr)
             rprint(file=sys.stderr)
             rprint("    If you want to make ArchiveBox run faster, disable specific archive methods instead:", file=sys.stderr)
-            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles", file=sys.stderr)
+            rprint("        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#plugins", file=sys.stderr)
             rprint(file=sys.stderr)
 
     @field_validator("CHECK_SSL_VALIDITY", mode="after")
@@ -825,6 +849,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +875,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")
@@ -1156,11 +1189,15 @@ def get_config(
     scope_overrides: ConfigPayload = {}
 
     if include_machine and machine is not None and machine.config:
-        from archivebox.machine.models import _sanitize_machine_config
+        machine_config = machine.config
+        if resolve_plugins:
+            from archivebox.machine.models import _sanitize_machine_config
+
+            machine_config = _sanitize_machine_config(machine_config, lib_dir=config_data.get("ABXPKG_LIB_DIR"))
 
         scope_overrides.update(
             normalize_runtime_config(
-                _sanitize_machine_config(machine.config, lib_dir=config_data.get("ABXPKG_LIB_DIR")),
+                machine_config,
                 only_crawl_execution=crawl_config_base,
                 exclude_runtime_derived=True,
                 json_safe=False,

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_snapshots.py

@@ -131,11 +131,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +992,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1539,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.request_delete()
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/forms.py

@@ -304,13 +304,13 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
             self.can_override_crawl_config = bool(user.is_authenticated and user.is_active and user.is_superuser)
         super().__init__(*args, **kwargs)
 
-        default_persona = Persona.get_or_create_default()
+        # Persona creation belongs to init/server startup, never a GET of /add/.
         persona_queryset = Persona.objects.order_by("name")
         if not self.can_override_crawl_config:
             persona_queryset = filter_personas_by_permissions(persona_queryset, {PERMISSIONS_PUBLIC})
         self.fields["persona"].queryset = persona_queryset
 
-        selected_persona = persona_queryset.filter(id=default_persona.id).first() or persona_queryset.first()
+        selected_persona = persona_queryset.filter(name__iexact="Default").first() or persona_queryset.first()
         default_config = get_config(persona=selected_persona) if selected_persona else get_config()
         if selected_persona:
             self.fields["persona"].initial = selected_persona.name
@@ -338,6 +338,12 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
                 get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
+    def clean_persona(self):
+        persona = self.cleaned_data.get("persona")
+        if persona:
+            persona.validate_name(persona.name)
+        return persona
+
     def clean(self):
         cleaned_data = super().clean() or {}
 

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -11,7 +12,7 @@ from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib import admin
-from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
+from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.db import IntegrityError, models, transaction
 from django.db.models import F, Q, QuerySet, Sum, Value
 from django.db.models.fields.json import KT
@@ -23,6 +24,7 @@ from django.utils.safestring import mark_safe
 from django.utils.text import slugify
 
 from archivebox.base_models.models import (
+    PagedQuerySet,
     ModelWithConfig,
     ModelWithDeleteAfter,
     ModelWithHealthStats,
@@ -172,9 +174,28 @@ class SnapshotTag(models.Model):
         unique_together: ClassVar[list[tuple[str, str]]] = [("snapshot", "tag")]
 
 
-class SnapshotQuerySet(models.QuerySet):
+class SnapshotQuerySet(PagedQuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def request_delete(self) -> int:
+        # Admin and API deletion share this durable command. Keep the row until
+        # the runner stops its writers and verifies removal from remote storage;
+        # RETRY_AT_MAX prevents capture scheduling while cleanup is pending.
+        return self.update(status=self.model.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
+
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -206,101 +227,6 @@ class SnapshotQuerySet(models.QuerySet):
                 truncate_overlong_charfields(obj)
         return super().bulk_create(objs, *args, **kwargs)
 
-    def paged_iterator(self, chunk_size: int = 500):
-        """
-        Iterate snapshots using bounded keyset pages instead of one streaming cursor.
-
-        Django's iterator(chunk_size=...) still keeps a single SQLite SELECT
-        cursor open until the full queryset is exhausted. That is fine for
-        read-only exports, but update/migration code does filesystem work and
-        writes while iterating; a long-lived read cursor there can stretch lock
-        waits across thousands of rows. This respects the queryset's existing
-        filters, order_by(), select_related(), and prefetch_related() state; if
-        no ordering is defined, it falls back to primary-key order.
-        """
-        pk_field = self.model._meta.pk.name
-        raw_ordering = tuple(self.query.order_by or self.model._meta.ordering or (pk_field,))
-
-        if any(not isinstance(term, str) or term == "?" for term in raw_ordering):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        ordering = []
-        for term in raw_ordering:
-            descending = term.startswith("-")
-            field_name = term[1:] if descending else term
-            if field_name == "pk":
-                field_name = pk_field
-            ordering.append(f"-{field_name}" if descending else field_name)
-
-        ordered_field_names = [term.removeprefix("-") for term in ordering]
-        try:
-            if any(self.model._meta.get_field(field_name).null for field_name in ordered_field_names):
-                offset = 0
-                while True:
-                    batch = list(self[offset : offset + chunk_size])
-                    if not batch:
-                        break
-                    yield from batch
-                    offset += chunk_size
-                return
-        except (AttributeError, FieldDoesNotExist):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
-        if not any(field_name in unique_field_names for field_name in ordered_field_names):
-            offset = 0
-            while True:
-                batch = list(self[offset : offset + chunk_size])
-                if not batch:
-                    break
-                yield from batch
-                offset += chunk_size
-            return
-
-        last_values = None
-        value_field_names = tuple(dict.fromkeys([*ordered_field_names, pk_field]))
-        while True:
-            batch_qs = self.order_by(*ordering)
-            if last_values is not None:
-                page_filter = models.Q()
-                for idx, term in enumerate(ordering):
-                    descending = term.startswith("-")
-                    field_name = term[1:] if descending else term
-                    prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
-                    comparison = "lt" if descending else "gt"
-                    page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
-                batch_qs = batch_qs.filter(page_filter)
-
-            batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
-            if not batch_rows:
-                break
-
-            pk_idx = value_field_names.index(pk_field)
-            snapshot_ids = [row[pk_idx] for row in batch_rows]
-            snapshots_by_id = {snapshot.pk: snapshot for snapshot in self.filter(pk__in=snapshot_ids).order_by()}
-
-            for row in batch_rows:
-                snapshot_id = row[pk_idx]
-                snapshot = snapshots_by_id.get(snapshot_id)
-                if snapshot is not None:
-                    yield snapshot
-
-            last_values = batch_rows[-1][: len(ordered_field_names)]
-
     # =========================================================================
     # Filtering Methods
     # =========================================================================
@@ -541,6 +467,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -574,7 +501,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -661,6 +588,49 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).request_delete()
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        # Finish the bounded read before touching files: even an open SELECT
+        # cursor can hold a SQLite read lock for the duration of slow cleanup.
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""
@@ -1073,7 +1043,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         from django.db import transaction
 
         def finish_snapshot_save():
-            self.reconcile_filesystem_links()
             crawl = Crawl.objects.filter(pk=self.crawl_id).first()
             if crawl is None:
                 return
@@ -1093,8 +1062,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             # Crawl.urls remains the original submitted source. Snapshot rows
             # are the normalized work queue and discovery projection.
 
-        # get_or_create/update_or_create wrap save() in atomic(); defer filesystem
-        # work and crawl maintenance so SQLite commits before touching the disk.
+        # get_or_create/update_or_create wrap save() in atomic(); attach inherited
+        # tags after that write commits. Filesystem projections belong to the runner.
         transaction.on_commit(finish_snapshot_save)
 
     # =========================================================================
@@ -1198,6 +1167,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         target_dir = self.get_storage_path_for_version(target)
         if target_dir.exists():
+            self.reconcile_with_index(output_dir=target_dir, update_existing_archive_results=False)
             self.hydrate_archiveresult_output_metadata(snapshot_dir=target_dir)
         if cleanup:
             old_dir, new_dir = cleanup
@@ -4773,9 +4743,13 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     @property
     def pwd(self) -> str:
-        """Working directory, derived from the snapshot/plugin path if the Process row is gone."""
+        """Working directory metadata without probing archive storage."""
         process = self.process_record
-        return process.pwd if process and process.pwd else str(self.output_dir)
+        return (
+            process.pwd
+            if process and process.pwd
+            else str(self.snapshot.get_storage_path_for_version(self.snapshot.fs_version) / self.plugin)
+        )
 
     @property
     def cmd(self) -> list:

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/core/takeover_util.py

@@ -62,7 +62,7 @@ def runtime_stack_owner(*, data_dir: str | Path, exclude_id=None):
         # not keep stealing leadership from the next foreground command.
         base_qs.filter(process_type=Process.TypeChoices.ORCHESTRATOR).exclude(parent__process_type=Process.TypeChoices.SUPERVISORD),
     ):
-        for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+        for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
             if proc.is_running:
                 return proc
             proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -95,7 +95,7 @@ def foreground_runner_owner(*, data_dir: str | Path, exclude_id=None):
     )
     if exclude_id is not None:
         qs = qs.exclude(id=exclude_id)
-    for proc in qs.order_by("-created_at", "-modified_at").iterator(chunk_size=50):
+    for proc in qs.order_by("-created_at", "-modified_at").paged_iterator(chunk_size=50):
         if proc.is_running:
             return proc
         proc.mark_exited(exit_code=proc.exit_code if proc.exit_code is not None else 0)
@@ -178,7 +178,7 @@ def live_runner_processes(*, data_dir: str | Path):
     qs = qs.filter(machine=machine)
     live = []
     foreign_namespace_ids = []
-    for process in qs.order_by("started_at", "created_at").iterator(chunk_size=20):
+    for process in qs.order_by("started_at", "created_at").paged_iterator(chunk_size=20):
         if not process.shares_pid_namespace:
             foreign_namespace_ids.append(process.id)
             continue

archivebox/machine/models.py

@@ -20,7 +20,7 @@ from django.utils.functional import cached_property
 
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
-from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
+from archivebox.base_models.models import PagedQuerySet, ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
@@ -255,40 +255,40 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Reconcile once, on the first Machine.current() call in this process.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
         # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine
@@ -890,7 +890,7 @@ class Binary(ModelWithHealthStats, ModelWithQueue):
 # =============================================================================
 
 
-class ProcessManager(models.Manager):
+class ProcessManager(models.Manager.from_queryset(PagedQuerySet)):
     """Manager for Process model."""
 
     def current(self) -> Process:
@@ -1640,9 +1640,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         if machine is not None:
             stale = stale.filter(machine=machine)
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # stale Process backlog cannot be materialized in memory at once.
-        for proc in stale.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in stale.order_by("pk").paged_iterator(chunk_size=100):
             shares_pid_namespace = proc.shares_pid_namespace
             if shares_pid_namespace and proc.poll() is not None:
                 cleaned += 1
@@ -2587,9 +2586,8 @@ class Process(ModelWithDeleteAfter, models.Model):
             status=cls.StatusChoices.RUNNING,
         )
 
-        # Recovery can run against damaged DB state; stream rows so a large
-        # orphaned Process backlog cannot be materialized in memory at once.
-        for proc in running_children.iterator(chunk_size=100):
+        # Close each bounded read before inspecting processes or writing their state.
+        for proc in running_children.order_by("pk").paged_iterator(chunk_size=100):
             if not proc.is_running:
                 proc.mark_exited(
                     exit_code=proc.exit_code if proc.exit_code is not None else _default_exit_code_for_unowned_process(proc.process_type),

archivebox/misc/checks.py

@@ -194,7 +194,10 @@ def check_not_root():
     if os.geteuid() == 0 and not (is_getting_help or is_getting_version):
         print("[yellow][!] Running ArchiveBox as root is not recommended.[/yellow]", file=sys.stderr)
         print("    Root-owned DATA_DIR files may be inaccessible to non-root users later.", file=sys.stderr)
-        print("        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root", file=sys.stderr)
+        print(
+            "        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user",
+            file=sys.stderr,
+        )
 
 
 def is_archivebox_source_root(path: Path | str | None = None) -> bool:
@@ -255,7 +258,7 @@ def check_data_dir_permissions(config=None, **config_kwargs):
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions]https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#permissions[/link]",
         )
         STDERR.print(
-            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid]https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid[/link]",
+            "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues[/link]",
         )
         STDERR.print(
             "    [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts]https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts[/link]",

archivebox/misc/shell_welcome_message.py

@@ -48,10 +48,10 @@ if __name__ == "__main__":
 
     prnt("[i] :heavy_dollar_sign: Welcome to the ArchiveBox Shell!")
     prnt(
-        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Shell-Usage[/link]",
+        "    [deep_sky_blue4]Docs:[/deep_sky_blue4] [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage]https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage[/link]",
     )
     prnt(
-        "          [link=https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/en/dev/apidocs/archivebox/archivebox.html[/link]",
+        "          [link=https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html]https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.html[/link]",
     )
     prnt()
     prnt(" :grey_question: [violet]Hint[/] [i]Here are some examples to get started:[/]")

archivebox/personas/forms.py

@@ -122,14 +122,14 @@ class PersonaAdminForm(PluginConfigFormMixin, forms.ModelForm):
                 "Use the custom path/CDP option if the browser data lives elsewhere."
             )
 
-        self.build_plugin_groups(get_config(persona=self.instance) if self.instance and self.instance.pk else get_config())
+        self.build_plugin_groups(get_config(persona=self.instance) if not self.instance._state.adding else get_config())
 
     def clean_name(self) -> str:
         name = str(self.cleaned_data.get("name") or "").strip()
         is_valid, error_message = validate_persona_name(name)
         if not is_valid:
             raise forms.ValidationError(error_message)
-        return name
+        return self.instance.validate_name(name)
 
     def clean(self) -> dict[str, Any]:
         cleaned_data = super().clean()

archivebox/personas/models.py

@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any
 from collections.abc import Mapping
 
 from django.db import IntegrityError, models
+from django.core.exceptions import ValidationError
 from django.db.models.fields.json import KT
 from django.conf import settings
 from django.utils import timezone
@@ -74,8 +75,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -117,6 +122,13 @@ class Persona(ModelWithConfig):
         app_label = "personas"
 
     def save(self, *args, **kwargs):
+        # Adopt the spelling on disk, never rename a Chrome profile just to fix case.
+        self.__dict__.pop("_persona_path", None)
+        canonical_name = self.validate_name(self.name)
+        if canonical_name != self.name:
+            self.name = canonical_name
+            if kwargs.get("update_fields") is not None:
+                kwargs["update_fields"] = tuple(dict.fromkeys([*kwargs["update_fields"], "name"]))
         config = dict(self.config or {})
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
@@ -133,10 +145,59 @@ class Persona(ModelWithConfig):
 
     @property
     def path(self) -> Path:
-        """Path to persona directory under PERSONAS_DIR."""
+        """Reuse the directory's spelling without renaming a Chrome profile."""
         from archivebox.config.constants import CONSTANTS
 
-        return CONSTANTS.PERSONAS_DIR / self.name
+        cached = self.__dict__.get("_persona_path")
+        if cached is None or cached[0] != self.name:
+            directory_name = self.directory_name(self.name)
+            cached = (self.name, CONSTANTS.PERSONAS_DIR / directory_name)
+            self.__dict__["_persona_path"] = cached
+        return cached[1]
+
+    @staticmethod
+    def directory_name(name: str) -> str:
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.personas.importers import validate_persona_name
+
+        valid, error = validate_persona_name(name)
+        if not valid:
+            raise ValidationError(error)
+        # Linux can hold both Default/default; macOS usually aliases them. Reuse a
+        # single match on either filesystem, but never guess which conflicting
+        # profile contains the user's login state. Only list names, not profile trees.
+        try:
+            with os.scandir(CONSTANTS.PERSONAS_DIR) as entries:
+                matches = sorted(entry.name for entry in entries if entry.name.casefold() == name.casefold())
+        except FileNotFoundError:
+            matches = []
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona directories: {'; '.join(str(CONSTANTS.PERSONAS_DIR / match) for match in matches)}. "
+                "Stop Chrome and rename the conflicting directories manually to distinct names; no profile directories were renamed.",
+            )
+        return matches[0] if matches else name
+
+    @classmethod
+    def find_named(cls, name: str) -> "Persona | None":
+        # SQLite's iexact is ASCII-only; Python casefold gives every entry point
+        # the same Unicode matching rules as the filesystem-name lookup above.
+        matches = [(pk, existing) for pk, existing in cls.objects.values_list("pk", "name") if existing.casefold() == name.casefold()]
+        if len(matches) > 1:
+            raise ValidationError(
+                f"Conflicting persona records: {'; '.join(f'{existing!r} (ID {pk})' for pk, existing in matches)}. "
+                "Rename the conflicting personas manually to distinct names; profile files have not been changed.",
+            )
+        return cls.objects.get(pk=matches[0][0]) if matches else None
+
+    def validate_name(self, name: str) -> str:
+        existing = self.find_named(name)
+        if existing is not None and existing.pk != self.pk:
+            raise ValidationError(
+                f"Name conflicts with persona {existing.name!r} (ID {existing.pk}, directory {existing.path}). "
+                "Choose a distinct name or rename that persona record manually.",
+            )
+        return self.directory_name(name)
 
     @property
     def CHROME_USER_DATA_DIR(self) -> str:
@@ -156,9 +217,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -171,10 +235,13 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
+        return derive_persona_config(name=self.path.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
+        # Recheck at profile preparation, including changes made outside this process.
+        self.__dict__.pop("_persona_path", None)
+        self.validate_name(self.name)
         self.path.mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_profile").mkdir(parents=True, exist_ok=True)
         (self.path / "chrome_downloads").mkdir(parents=True, exist_ok=True)
@@ -246,16 +313,26 @@ class Persona(ModelWithConfig):
                     fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
 
     @classmethod
-    def get_or_create_named(cls, name: str) -> "Persona":
+    def get_or_create_named(cls, name: str, *, defaults: dict | None = None) -> "Persona":
+        # Init/startup must reuse a user-created 'default', not create a second
+        # DB row pointing at a different (or shared) profile (issue #1896).
         persona_name = (name or "Default").strip() or "Default"
-        persona = cls.objects.filter(name=persona_name).first()
+        persona = cls.find_named(persona_name)
         if persona is not None:
+            canonical_name = persona.validate_name(persona.name)
+            if canonical_name != persona.name:
+                persona.name = canonical_name
+                persona.save(update_fields=["name"])
             return persona
 
         try:
-            return cls.objects.create(name=persona_name)
-        except IntegrityError:
-            return cls.objects.get(name=persona_name)
+            return cls.objects.create(name=persona_name, **(defaults or {}))
+        except (IntegrityError, ValidationError):
+            persona = cls.find_named(persona_name)
+            if persona is None:
+                raise
+            persona.validate_name(persona.name)
+            return persona
 
     def runtime_root_for_crawl(self, crawl) -> Path:
         return Path(crawl.output_dir) / ".persona" / self.name
@@ -315,20 +392,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +424,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/services/runner.py

@@ -65,6 +65,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -738,8 +739,16 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(machine.config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
+        Path(self.crawl_output_dir).mkdir(parents=True, exist_ok=True)
         if self.persona:
             self.base_config.update(
                 self.persona.prepare_runtime_for_crawl(
@@ -948,6 +957,7 @@ class CrawlRunner:
                     },
                 )
         snapshot_output_dir = str(snapshot.output_dir)
+        Path(snapshot_output_dir).mkdir(parents=True, exist_ok=True)
         tags = snapshot.tags_str()
         config = self.base_config.for_crawl_runtime(
             crawl=snapshot.crawl,
@@ -1245,6 +1255,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1260,6 +1271,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1283,6 +1295,7 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
                 if self.execution_interrupted():
                     return
                 if snapshot["status"] == "sealed":
@@ -1306,6 +1319,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1981,14 +2004,35 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read

archivebox/services/snapshot_service.py

@@ -146,7 +146,7 @@ class SnapshotService(BaseService):
             elif snapshot.status not in (Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED):
                 return
             if snapshot.status == Snapshot.StatusChoices.STARTED:
-                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+                await sync_to_async(snapshot.reconcile_filesystem_links, thread_sensitive=True)()
             retry_plugins = [str(name).strip() for name in (snapshot.config or {}).get("RETRY_PLUGINS", []) if str(name).strip()]
             async with self._ownership_lock:
                 self._run_ownership[str(event.snapshot_id)] = (str(event.event_id), snapshot.retry_at, was_sealed, retry_plugins)

archivebox/templates/core/setup_wizard.html

@@ -35,7 +35,7 @@
       <div class="abx-question-options" id="archivebox-setup-dns-options">
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="localhost"> <span><b>⭐ Localhost</b><small>No setup needed. <code id="archivebox-setup-localhost-dns-example">*.archivebox.localhost</code> works directly in modern browsers with all features.</small></span></label>
         <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="single"> <span><b>⚠️ Single-domain DNS</b><small>Register a DNS record like <code>A archivebox.example.com</code> → <code>your server's IP</code>, or add an equivalent <code>/etc/hosts</code> entry pointing to your server. Some archived content that depends on JavaScript may not display properly in this mode. JavaScript still runs during capture, but viewing HTML captured by <code>wget</code>, <code>dom</code>, <code>responses</code>, <code>staticfile</code>, and similar plugins will not replay JavaScript unless wildcard DNS is used.</small></span></label>
-        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
+        <label class="abx-question-option"><input type="radio" name="archivebox-dns-mode" value="wildcard"> <span><b>⭐ Wildcard DNS</b><small>Point both the bare <code>BASE_URL</code> hostname and <code>*.BASE_URL</code> to this server using A/AAAA records or CNAME records. Ideal: safest isolation with all replay features. <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer">Setup instructions</a>.</small></span></label>
       </div>
       <div class="abx-question-status" id="archivebox-setup-dns-status">Choose a DNS mode to begin testing.</div>
     </fieldset>

archivebox/templates/core/system_warnings_banner.html

@@ -27,7 +27,7 @@
       Only <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ free_gb }} GiB</code>
       free on <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">DATA_DIR</code>
       — new archive jobs will fail until space is reclaimed.
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Disk-Usage" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#purging-entries" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">docs</a>
     {% elif mode == "high_memory" %}
       Virtual memory at <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ mem_pct }}%</code>
       — the host is close to OOM. Consider stopping crawls or scaling up.
@@ -35,7 +35,7 @@
       15-min loadavg <code style="background:#000;color:#fff;padding:1px 6px;border-radius:3px;font:700 11px/1.3 ui-monospace,Menlo,Consolas,monospace;text-transform:none;">{{ load_15 }}</code>
     {% elif mode == "unsafe" %}
       ArchiveBox single-domain mode — archived pages share an origin with this site
-      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Modes" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
+      <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#server_security_mode" target="_blank" rel="noopener noreferrer" style="color:#fff;text-decoration:underline;margin-left:6px;">why?</a>
     {% elif mode == "base_url_mismatch" %}
       Editing disabled from unrecognized URL. Visit
       <a href="{{ configured_base_url }}" style="color:#fff;text-decoration:underline;text-transform:none;">{{ configured_base_url }}</a>

archivebox/tests/conftest.py

@@ -32,6 +32,13 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
 SESSION_DATA_DIR = Path(
     os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
 ).resolve()
+if worker_id := os.environ.get("PYTEST_XDIST_WORKER"):
+    # Workers inherit the coordinator's environment. Give each one its own
+    # collection and cleanup root so a finishing worker cannot delete another's.
+    SESSION_DATA_DIR /= worker_id
+    for runtime_key in ("ABXPKG_LIB_DIR", "PERSONAS_DIR", "CHROME_USER_DATA_DIR"):
+        if runtime_path := os.environ.get(runtime_key):
+            os.environ[runtime_key] = str(Path(runtime_path) / worker_id)
 
 os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
 os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
@@ -437,7 +444,7 @@ def hermetic_lib_dir(tmp_path):
 @pytest.fixture
 def cached_abxpkg_lib_dir():
     """Reuse one real abxpkg installation cache for the current pytest session."""
-    lib_dir = SESSION_DATA_DIR / "lib"
+    lib_dir = Path(os.environ["ABXPKG_LIB_DIR"]) if os.environ.get("ABXPKG_LIB_DIR") else SESSION_DATA_DIR / "lib"
     lib_dir.mkdir(parents=True, exist_ok=True)
     original_lib_dir = os.environ.get("ABXPKG_LIB_DIR")
     os.environ["ABXPKG_LIB_DIR"] = str(lib_dir)

archivebox/tests/test_api_discovery_performance.py

@@ -0,0 +1,54 @@
+"""Server discovery must not repeat schema generation or probe installed tools."""
+
+import sys
+import threading
+import os
+from pathlib import Path
+
+import pytest
+from django.test.utils import override_script_prefix
+
+from archivebox.config import VERSION
+from archivebox.machine.models import Machine
+from archivebox.tests.conftest import install_real_binary, resolve_abxpkg_binary_env
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("script_name", ["", "/mounted"])
+def test_repeated_discovery_reuses_schema_without_binary_filesystem_probes(client, hermetic_lib_dir, script_name):
+    install_real_binary("node", binproviders="env,apt,brew")
+    resolve_abxpkg_binary_env(hermetic_lib_dir, "node")
+    machine = Machine.current()
+    machine.config = {**machine.config, "NODE_BINARY": str(hermetic_lib_dir / "env" / "bin" / "node")}
+    machine.save(update_fields=["config"])
+    request = {"HTTP_HOST": "api.archivebox.localhost:5797", "SCRIPT_NAME": script_name}
+    with override_script_prefix(script_name):
+        first = client.get("/api/v1/openapi.json", **request)
+    assert first.status_code == 200, first.content
+    schema_calls = []
+    filesystem_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name == "model_json_schema":
+            schema_calls.append(frame.f_code.co_filename)
+        if (event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}) or (
+            event == "c_call" and arg in (os.stat, os.lstat)
+        ):
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(hermetic_lib_dir):
+                filesystem_calls.append(str(path))
+
+    previous = sys.getprofile()
+    previous_thread = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        with override_script_prefix(script_name):
+            second = client.get("/api/v1/openapi.json", **request)
+    finally:
+        threading.setprofile_all_threads(previous_thread)
+        sys.setprofile(previous)
+    assert second.status_code == 200, second.content
+    assert second.json() == first.json()
+    assert second.json()["info"]["version"] == VERSION
+    assert f"{script_name}/api/v1/core/snapshots" in second.json()["paths"]
+    assert (len(schema_calls), filesystem_calls) == (0, [])

archivebox/tests/test_api_v1_cli_add.py

@@ -478,3 +478,40 @@ def test_api_cli_add_rejects_file_path_and_shell_injection_payloads(tmp_path):
     with use_archivebox_db(tmp_path):
         tag_names = set(SnapshotTag.objects.filter(snapshot=snapshot).values_list("tag__name", flat=True))
     assert "api-security" in tag_names
+
+
+def test_api_add_only_queues_work_without_storage_or_worker_control(client, api_headers):
+    import sys
+    from archivebox.config import CONSTANTS
+
+    calls = []
+
+    def observe(frame, event, arg):
+        if event != "call":
+            return
+        name = frame.f_code.co_name
+        if name == "ensure_background_runner":
+            calls.append(name)
+        if name in {"stat", "lstat", "exists", "is_dir", "is_file", "is_symlink", "mkdir", "iterdir", "symlink_to"}:
+            path = frame.f_locals.get("self")
+            if isinstance(path, Path) and path.is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                calls.append((name, str(path)))
+
+    previous = sys.getprofile()
+    sys.setprofile(observe)
+    try:
+        response = client.post(
+            "/api/v1/cli/add",
+            data={"urls": ["https://example.com/queue-only"], "tag": "queue-only"},
+            content_type="application/json",
+            **api_headers,
+        )
+    finally:
+        sys.setprofile(previous)
+    assert response.status_code == 200, response.content
+    crawl = Crawl.objects.get(pk=response.json()["result"]["crawl_id"])
+    assert crawl.urls == "https://example.com/queue-only"
+    assert crawl.status == Crawl.StatusChoices.QUEUED
+    assert crawl.retry_at is not None
+    assert not crawl.snapshot_set.exists()
+    assert calls == []

archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py

@@ -477,25 +477,38 @@ def test_paused_snapshot_survives_server_restart_and_resumes_via_api(client, tmp
         stop_server(tmp_path)
 
 
-def test_rest_snapshot_delete_removes_output_dir(client, api_headers):
-    url = "https://example.com/delete-path-snapshot"
+@pytest.mark.timeout(180)
+def test_rest_snapshot_delete_removes_output_dir(tmp_path, recursive_test_site):
+    import time
+    from .test_snapshot_service import _snapshot_state
+    from .conftest import create_admin_and_token, live_api_request, start_archivebox_server, stop_archivebox_process
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/snapshots",
-        payload={"url": url, "depth": 0, "status": Snapshot.StatusChoices.QUEUED},
-        headers=api_headers,
-    )
-    assert response.status_code == 200, response.content.decode()
-
-    snapshot = Snapshot.objects.get(url=url)
-    snapshot_dir = Path(snapshot.output_dir)
-    snapshot_dir.mkdir(parents=True, exist_ok=True)
-    (snapshot_dir / "delete-path-test.txt").write_text("snapshot output")
-    assert snapshot_dir.exists()
-
-    response = client.delete(f"/api/v1/core/snapshot/{snapshot.id}", **api_headers)
-    assert response.status_code == 200, response.content.decode()
-    assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
-    assert not snapshot_dir.exists()
+    init_archive(tmp_path)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=tmp_path, env=env)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
+    state = _snapshot_state(tmp_path, url)
+    assert any("Root" in path.read_text(errors="ignore") for path in (state["snapshot_dir"] / "wget").rglob("*.html"))
+    with use_archivebox_db(tmp_path):
+        snapshot_id = str(Snapshot.objects.get(url=url).pk)
+    token = create_admin_and_token(tmp_path)
+    server = start_archivebox_server(tmp_path, port=port, env=env, log_name="api-delete-handoff.log")
+    try:
+        response = live_api_request(port, "delete", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert response.status_code == 200, response.text
+        assert response.json()["queued_count"] == 1
+        assert response.json()["deleted_count"] == 0
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(tmp_path):
+            while Snapshot.objects.filter(pk=snapshot_id).exists() and time.monotonic() < deadline:
+                assert server.poll() is None
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+        assert not state["snapshot_dir"].exists()
+        assert not state["crawl_link"].is_symlink()
+        missing = live_api_request(port, "get", f"/api/v1/core/snapshot/{snapshot_id}", api_token=token)
+        assert missing.status_code == 404
+    finally:
+        stop_archivebox_process(server)

archivebox/tests/test_api_v1_core_snapshots.py

@@ -13,6 +13,65 @@ from archivebox.crawls.models import Crawl
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_browser_metadata_respects_only_new_and_reports_capture_owner(client, api_headers):
+    url = "https://example.com/browser-deduplication"
+    for name in ("Original", "Replacement"):
+        persona = client.post(
+            "/api/v1/personas/sync",
+            data={"extension_persona_id": f"dedupe-{name}", "name": name},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert persona.status_code == 200, persona.content
+
+    def submit(only_new, persona):
+        queued = client.post(
+            "/api/v1/cli/add",
+            data={"urls": [url], "only_new": only_new, "persona": persona},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert queued.status_code == 200, queued.content
+        crawl_id = queued.json()["result"]["crawl_id"]
+        metadata = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": crawl_id},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert metadata.status_code == 200, metadata.content
+        return crawl_id, metadata.json()
+
+    first_crawl, first = submit(True, "Original")
+    second_crawl, reused = submit(True, "Replacement")
+    assert second_crawl != first_crawl
+    assert reused["id"] == first["id"]
+    assert reused["crawl_id"] == first_crawl
+    assert reused["persona"] == "Original"
+    assert Snapshot.objects.filter(url=url).count() == 1
+    forced_crawl, forced = submit(False, "Replacement")
+    assert forced["id"] != first["id"]
+    assert forced["crawl_id"] == forced_crawl
+    assert forced["persona"] == "Replacement"
+    assert Snapshot.objects.filter(url=url).count() == 2
+
+
+def test_snapshot_delete_only_queues_cleanup(client, api_headers):
+    created = client.post(
+        "/api/v1/core/snapshots",
+        data={"url": "https://example.com/queued-delete"},
+        content_type="application/json",
+        **api_headers,
+    )
+    assert created.status_code == 200, created.content
+    snapshot_id = created.json()["id"]
+    response = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+    assert response.status_code == 200, response.content
+    assert response.json()["queued_count"] == 1
+    assert response.json()["deleted_count"] == 0
+    assert Snapshot.objects.get(pk=snapshot_id).status == Snapshot.DELETING_STATE
+
+
 def test_snapshots_api_filters_status_column(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(
         urls="https://example.com",
@@ -150,3 +209,81 @@ def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_
     assert response.status_code == 200, response.content
     assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
     assert elapsed < 1
+
+
+def test_snapshot_metadata_requests_do_not_touch_archive_storage(client, api_admin_user, api_headers):
+    """Metadata writes and serialization must work without replay storage access."""
+    import sys
+    import threading
+    from pathlib import Path
+    from archivebox.config import CONSTANTS
+
+    url = "https://example.com/metadata-without-storage"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    storage_calls = []
+
+    def observe(frame, event, arg):
+        if event == "call" and frame.f_code.co_name in {
+            "stat",
+            "lstat",
+            "exists",
+            "is_dir",
+            "is_file",
+            "is_symlink",
+            "open",
+            "mkdir",
+            "iterdir",
+            "glob",
+            "rglob",
+            "readlink",
+            "symlink_to",
+            "unlink",
+        }:
+            path = frame.f_locals.get("self", frame.f_locals.get("path"))
+            if isinstance(path, (str, Path)) and Path(path).is_relative_to(CONSTANTS.ARCHIVE_DIR):
+                storage_calls.append((frame.f_code.co_name, str(path)))
+
+    previous = sys.getprofile()
+    thread_profile = threading.getprofile()
+    threading.setprofile_all_threads(observe)
+    try:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Browser title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert response.status_code == 200, response.content
+        snapshot_id = response.json()["id"]
+        updated = client.post(
+            "/api/v1/core/snapshots",
+            data={"url": url, "crawl_id": str(crawl.id), "title": "Updated title", "tags": ["browser"]},
+            content_type="application/json",
+            **api_headers,
+        )
+        assert updated.status_code == 200, updated.content
+        result = client.post(
+            "/api/v1/core/archiveresults",
+            data={"snapshot_id": snapshot_id, "plugin": "chrome_extension_viewport", "status": "started"},
+            **api_headers,
+        )
+        assert result.status_code == 200, result.content
+        result_id = result.json()["id"]
+        result_get = client.get(f"/api/v1/core/archiveresult/{result_id}", **api_headers)
+        assert result_get.status_code == 200, result_get.content
+        assert result_get.json()["output_files"] == {}
+        fetched = client.get(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert fetched.status_code == 200, fetched.content
+        assert fetched.json()["archiveresults"][0]["id"] == result_id
+        deleted = client.delete(f"/api/v1/core/snapshot/{snapshot_id}", **api_headers)
+        assert deleted.status_code == 200, deleted.content
+        assert deleted.json()["queued_count"] == 1
+    finally:
+        threading.setprofile_all_threads(thread_profile)
+        sys.setprofile(previous)
+    snapshot = Snapshot.objects.get(pk=snapshot_id)
+    assert snapshot.title == "Updated title"
+    assert snapshot.status == Snapshot.DELETING_STATE
+    assert list(snapshot.tags.values_list("name", flat=True)) == ["browser"]
+    assert fetched.json()["title"] == "Updated title"
+    assert storage_calls == []

archivebox/tests/test_binary_service.py

@@ -198,21 +198,26 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     assert Path(relibbed.abspath).resolve() == Path(host_binary).resolve()
 
 
-def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
-    name = "http"
+def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive):
+    name = "httpie"
     provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
     runtime_env = _runtime_env(initialized_archive)
 
-    _cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-
-    assert returncode != 0, stdout + stderr
-    assert "Binary http installation failed" in stderr
+    failed = run_archivebox_cmd(
+        ["install", "--binproviders=env", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert failed.returncode != 0, failed.stdout + failed.stderr
+    assert f"Binary {name} installation failed" in failed.stderr
 
     with use_archivebox_db(initialized_archive):
         queued = Binary.objects.get(name=name)
         queued_id = str(queued.id)
-        failed_process = Process.objects.filter(process_type=Process.TypeChoices.BINARY).latest("created_at")
+        failed_process = Process.objects.filter(binary_id=queued_id).latest("created_at")
         machine_config = Machine.objects.get(pk=queued.machine_id).config or {}
 
     assert queued.status == Binary.StatusChoices.QUEUED
@@ -223,28 +228,31 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert f"{name.upper().replace('-', '_')}_BINARY" not in machine_config
     assert not (provider_bin_dir / name).exists()
 
-    with use_archivebox_db(initialized_archive):
-        queued = Binary.objects.get(pk=queued_id)
-        queued.binproviders = "pip"
-        queued.overrides = {"pip": {"install_args": ["httpie>=3.2.4"]}}
-        queued.save(update_fields=["binproviders", "overrides", "modified_at"])
-    recovered_runtime_env = _runtime_env(initialized_archive)
-
-    _cmd_result = run_archivebox_cmd(
-        ["run", f"--binary-id={queued_id}"],
+    recovered_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip", name],
         cwd=initialized_archive,
         timeout=120,
-        env=recovered_runtime_env,
+        env=runtime_env,
         default_cli_env=True,
         disable_extractors=True,
     )
-    recover_stdout, recover_stderr, recover_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-
-    assert recover_code == 0, recover_stdout + recover_stderr
+    assert recovered_install.returncode == 0, recovered_install.stdout + recovered_install.stderr
+    repeated_install = run_archivebox_cmd(
+        ["install", "--binproviders=pip", name],
+        cwd=initialized_archive,
+        timeout=120,
+        env=runtime_env,
+        default_cli_env=True,
+        disable_extractors=True,
+    )
+    assert repeated_install.returncode == 0, repeated_install.stdout + repeated_install.stderr
     with use_archivebox_db(initialized_archive):
         recovered = Binary.objects.get(pk=queued_id)
+        # Providers may resolve their own dependencies during installation. Those
+        # Process rows are not attempts to install this binary; track its exact
+        # identity instead of assuming the last two global rows belong to it.
         process_exit_codes = list(
-            Process.objects.filter(process_type=Process.TypeChoices.BINARY).order_by("created_at").values_list("exit_code", flat=True),
+            Process.objects.filter(binary_id=queued_id).order_by("created_at").values_list("exit_code", flat=True),
         )
 
     assert recovered.status == Binary.StatusChoices.INSTALLED
@@ -253,4 +261,4 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     assert Path(recovered.abspath) == provider_bin_dir / name
     assert recovered.binprovider == "pip"
     assert Path(recovered.abspath).is_file()
-    assert process_exit_codes[-2:] == [1, 0]
+    assert process_exit_codes == [1, 0]

archivebox/tests/test_cli_add_4.py

@@ -325,11 +325,15 @@ def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
     assert snapshots == [("https://example.com", 0), ("https://example.com", 0)]
 
 
-def test_snapshot_create_creates_current_output_directory(initialized_archive):
-    """Test the user-facing snapshot creation path creates an output directory."""
-    env = cli_env(disable_extractors=True)
+def test_snapshot_create_server_creates_current_output_directory(initialized_archive, recursive_test_site):
+    """Queued metadata gets its output directory from the normal server worker."""
+    from .conftest import start_archivebox_server, stop_archivebox_process, get_free_port, wait_for_snapshot_capture
+
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
     run_archivebox_cmd(
-        ["snapshot", "create", "https://example.com"],
+        ["snapshot", "create", url],
         cwd=initialized_archive,
         env=env,
         check=True,
@@ -338,6 +342,12 @@ def test_snapshot_create_creates_current_output_directory(initialized_archive):
     with use_archivebox_db(initialized_archive):
         snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get())
 
+    assert find_snapshot_dir(initialized_archive, snapshot_id) is None
+    server = start_archivebox_server(initialized_archive, env=env, port=port, log_name="snapshot-output-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(initialized_archive, url, timeout=120)
+    finally:
+        stop_archivebox_process(server)
     snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
     assert snapshot_dir.is_dir()

archivebox/tests/test_cli_background_screenshot.py

@@ -0,0 +1,39 @@
+"""Exercise the browser target lifecycle through a real multi-page CLI crawl."""
+
+import os
+import sqlite3
+import subprocess
+
+import pytest
+
+
+@pytest.mark.timeout(180)
+def test_cli_crawl_captures_background_screenshots(tmp_path):
+    env = {**os.environ, "SCREENSHOT_TIMEOUT": "5"}
+    initialized = subprocess.run(
+        ["archivebox", "init", "--quick"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=60,
+    )
+    assert initialized.returncode == 0, initialized.stderr
+    captured = subprocess.run(
+        ["archivebox", "add", "--plugins=title,wget,screenshot", "https://example.com", "https://example.org"],
+        cwd=tmp_path,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+    )
+    assert captured.returncode == 0, captured.stderr
+    with sqlite3.connect(tmp_path / "index.sqlite3") as db:
+        results = db.execute("SELECT status, output_str FROM core_archiveresult WHERE plugin='screenshot'").fetchall()
+    assert len(results) == 2, captured.stdout
+    if any(status != "succeeded" for status, _ in results):
+        print(captured.stdout, captured.stderr)
+    assert all(status == "succeeded" for status, _ in results), (results, captured.stdout, captured.stderr)
+    images = list((tmp_path / "archive" / "users").glob("*/snapshots/**/screenshot/screenshot.png"))
+    assert len(images) == 2
+    assert all(image.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") and image.stat().st_size > 1000 for image in images)

archivebox/tests/test_cli_list.py

@@ -26,9 +26,7 @@ def test_static_export_creates_detail_page_for_unarchived_snapshot(snapshot):
     from archivebox.config import CONSTANTS
 
     snapshot_dir = Path(snapshot.output_dir)
-    assert snapshot_dir.is_dir()
-    assert not any(snapshot_dir.iterdir())
-    snapshot_dir.rmdir()
+    # Queued metadata has no output directory until an explicit writer runs.
     assert not snapshot_dir.exists()
 
     html = Snapshot.objects.filter(pk=snapshot.pk).to_html(with_headers=True)

archivebox/tests/test_cli_persona.py

@@ -50,12 +50,14 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
     """Import an actual Chromium-created session and archive its private page."""
     import hashlib
     import json
+    import shlex
     import subprocess
 
     from werkzeug.wrappers import Response
     from archivebox.tests.conftest import resolve_abxpkg_chrome_env
 
     browser_env = resolve_abxpkg_chrome_env(tmp_path / "lib")
+    browser_env["ABXPKG_LIB_DIR"] = str(tmp_path / "lib")
     source = tmp_path / "browser"
     httpserver.expect_request("/login").respond_with_data(
         '<html><title>Signed in</title><script>localStorage.setItem("persona-preference", "forest-green")</script>Signed in</html>',
@@ -90,6 +92,7 @@ const puppeteer = require(process.argv[1]);
         const page = await browser.newPage();
         await page.goto(process.argv[4]);
         console.log(await page.title());
+        console.log(await page.evaluate(() => localStorage.getItem('persona-preference')));
     } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exit(1); });
 """,
@@ -104,6 +107,7 @@ const puppeteer = require(process.argv[1]);
     )
     assert seeded.returncode == 0, seeded.stderr
     assert "Signed in" in seeded.stdout
+    assert "forest-green" in seeded.stdout
     preferences = source / "Profile 2" / "Preferences"
     original_preferences = preferences.read_bytes()
 
@@ -161,6 +165,9 @@ const puppeteer = require(process.argv[1]);
         env={**browser_env, "CHROME_SANDBOX": "false"},
     )
     assert archived.returncode == 0, archived.stderr
+    launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
+    assert len(launch_commands) == 1
+    assert shlex.split(launch_commands[0].read_text().splitlines()[1])[0] == browser_env["CHROME_BINARY"]
     outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
     assert len(outputs) == 1
     html = outputs[0].read_text()

archivebox/tests/test_cli_server.py

@@ -448,7 +448,10 @@ def test_port_compatibility_does_not_add_listeners_outside_docker_defaults(tmp_p
         check=True,
     )
     workers = json.loads(result.stdout)
-    assert [worker["name"] for worker in workers] == ["worker_runserver" if debug else "worker_daphne", "worker_runner"]
+    assert [worker["name"] for worker in workers] == [
+        "worker_runserver" if debug else "worker_daphne",
+        "worker_runner",
+    ]
     command = shlex.split(workers[0]["command"])
     assert "--endpoint" not in command
     assert (f"127.0.0.1:{port}" if debug else f"--port={port}") in command

archivebox/tests/test_cli_snapshot.py

@@ -20,6 +20,10 @@ from archivebox.tests.conftest import (
     create_test_url,
     parse_jsonl_output,
     run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    get_free_port,
+    wait_for_snapshot_capture,
 )
 from archivebox.tests.test_orm_helpers import use_archivebox_db
 
@@ -380,19 +384,30 @@ class TestSnapshotDelete:
         assert "Would delete" in stderr
 
 
-def test_snapshot_creates_snapshot_with_correct_url(tmp_path, initialized_archive):
+def test_snapshot_creates_snapshot_with_correct_url(tmp_path, initialized_archive, recursive_test_site):
     """Test that snapshot stores the exact URL in the database."""
-    env = cli_env(disable_extractors=True)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget")
+    url = recursive_test_site["root_url"]
 
-    run_archivebox_cmd(
-        ["snapshot", "create", "https://example.com"],
+    result = run_archivebox_cmd(
+        ["snapshot", "create", url],
         cwd=tmp_path,
         env=env,
     )
+    assert result.returncode == 0, result.stderr
 
     with use_archivebox_db(tmp_path):
-        snapshot = Snapshot.objects.select_related("crawl__created_by").get(url="https://example.com")
+        snapshot = Snapshot.objects.select_related("crawl__created_by").get(url=url)
         username = snapshot.crawl.created_by.username
+        assert snapshot.status == Snapshot.StatusChoices.QUEUED
+        assert not snapshot.output_dir.exists()
+
+    server = start_archivebox_server(tmp_path, env=env, port=port, log_name="snapshot-create-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(tmp_path, url, timeout=120)
+    finally:
+        stop_archivebox_process(server)
 
     # Verify the crawl tree contains a relative symlink to the user-scoped snapshot output.
     snapshots_root = tmp_path / "archive" / "users" / username / "snapshots"

archivebox/tests/test_cli_status.py

@@ -107,22 +107,22 @@ def test_status_detects_orphaned_directories(initialized_archive):
     assert "archivebox update" in result.stdout
 
 
-def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive):
+def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive, recursive_test_site):
     """Test status reads archived/present counts from the current snapshot output layout."""
     env = cli_env(disable_extractors=True)
     env = env.copy()
     env["ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS"] = "true"
 
-    _create_snapshot_rows(initialized_archive, env, "https://example.com")
+    url = recursive_test_site["root_url"]
+    captured = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=initialized_archive, env=env, timeout=120)
+    assert captured.returncode == 0, captured.stdout + captured.stderr
 
     with use_archivebox_db(initialized_archive):
-        snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url="https://example.com")
+        snapshot_id = Snapshot.objects.values_list("id", flat=True).get(url=url)
 
     snapshot_dir = find_snapshot_dir(initialized_archive, str(snapshot_id))
     assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
-    title_dir = snapshot_dir / "title"
-    title_dir.mkdir(parents=True, exist_ok=True)
-    (title_dir / "title.txt").write_text("Example Domain")
+    assert any("Root" in path.read_text(errors="ignore") for path in (snapshot_dir / "wget").rglob("*.html"))
 
     result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env)
 

archivebox/tests/test_cli_update.py

@@ -19,6 +19,50 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_rescan_recovers_already_migrated_idless_result(initialized_archive, tmp_path):
+    """Repair collections already migrated by versions affected by #1899."""
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(["add", "--index-only", "https://example.com/legacy-result"], env=env, check=True)
+    run_queued_crawls(initialized_archive, env)
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url="https://example.com/legacy-result")
+        snapshot.write_index_jsonl()
+        output_dir = snapshot.output_dir
+        snapshot_id = snapshot.pk
+    payload = b"<html>Recovered after a previous migration</html>"
+    (output_dir / "singlefile.html").write_bytes(payload)
+    with (output_dir / "index.jsonl").open("a") as index:
+        index.write(
+            json.dumps(
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(snapshot_id),
+                    "plugin": "singlefile",
+                    "hook_name": "",
+                    "status": "succeeded",
+                    "output_str": "singlefile.html",
+                    "start_ts": "2021-03-07T23:56:16-08:00",
+                },
+            )
+            + "\n",
+        )
+    result_id = None
+    for expected in ("Repaired: 1", "Unchanged: 1"):
+        result = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert result.returncode == 0, result.stdout + result.stderr
+        assert expected in result.stdout
+        with use_archivebox_db(initialized_archive):
+            archived = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="singlefile")
+            assert archived.status == "succeeded"
+            assert archived.output_size == len(payload)
+            assert archived.output_files
+            assert archived.snapshot.output_size == len(payload)
+            if result_id is not None:
+                assert archived.pk == result_id
+            result_id = archived.pk
+        assert (output_dir / "singlefile.html").read_bytes() == payload
+
+
 def test_rescan_imports_current_layout_and_is_idempotent(initialized_archive, tmp_path):
     env = cli_env(disable_extractors=True)
     run_archivebox_cmd(["add", "--index-only", "https://example.com/rescan"], cwd=initialized_archive, env=env, check=True)
@@ -284,3 +328,39 @@ def test_update_seals_migrated_snapshots(initialized_archive):
         status = Snapshot.objects.values_list("status", flat=True).get()
 
     assert status == "sealed"
+
+
+@pytest.mark.parametrize("captured", [False, True])
+def test_index_only_distinguishes_uncaptured_metadata_from_missing_archive(initialized_archive, recursive_test_site, captured):
+    url = recursive_test_site["root_url"]
+    env = cli_env(disable_extractors=True)
+    if captured:
+        result = run_archivebox_cmd(["add", "--plugins=wget", url], cwd=initialized_archive, env=env, timeout=120)
+        assert result.returncode == 0, result.stdout + result.stderr
+    else:
+        created = run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, env=env, check=True)
+        sealed = run_archivebox_cmd(
+            ["snapshot", "update", "--status=sealed"],
+            cwd=initialized_archive,
+            env=env,
+            input=created.stdout,
+        )
+        assert sealed.returncode == 0, sealed.stdout + sealed.stderr
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url=url)
+        directory = snapshot.output_dir
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        if captured:
+            assert snapshot.downloaded_at is not None
+            assert snapshot.archiveresult_set.filter(plugin="wget", status="succeeded").exists()
+            directory.rename(directory.with_name(directory.name + ".removed"))
+        else:
+            assert snapshot.downloaded_at is None
+            assert not snapshot.archiveresult_set.exists()
+        assert not directory.exists()
+    result = run_archivebox_cmd(["update", "--index-only"], cwd=initialized_archive, env=env, timeout=120)
+    if captured:
+        assert result.returncode != 0
+        assert "Missing snapshot directory" in result.stdout + result.stderr
+    else:
+        assert result.returncode == 0, result.stdout + result.stderr

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -15,6 +15,49 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.parametrize("index_format", ["history", "archive_results"])
+def test_rescan_imports_legacy_result_rows_and_saved_content(tmp_path, initialized_archive, index_format):
+    """Issue #1899: orphan metadata must become visible results on the first import."""
+    env = cli_env(disable_extractors=True)
+    legacy_dir = tmp_path / "archive" / "1615190176"
+    legacy_dir.mkdir(parents=True)
+    payload = b"<html><title>Recovered capture</title><body>saved content</body></html>"
+    (legacy_dir / "singlefile.html").write_bytes(payload)
+    entry = {
+        "plugin": "singlefile",
+        "status": "succeeded",
+        "output": "singlefile.html",
+        "start_ts": "2021-03-07T23:56:16-08:00",
+        "end_ts": "2021-03-07T23:56:16-08:00",
+    }
+    metadata = {
+        "url": "https://example.com/recovered",
+        "timestamp": legacy_dir.name,
+        "title": "Recovered capture",
+        index_format: {"singlefile": [entry]} if index_format == "history" else [entry],
+    }
+    original_json = json.dumps(metadata)
+    (legacy_dir / "index.json").write_text(original_json)
+    result_id = None
+    for _ in range(2):
+        command = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert command.returncode == 0, command.stdout + command.stderr
+        with use_archivebox_db(tmp_path):
+            snapshot = Snapshot.objects.get(url=metadata["url"])
+            result = snapshot.archiveresult_set.get(plugin="singlefile")
+            assert result.status == "succeeded"
+            assert result.output_size == len(payload)
+            assert result.output_files
+            assert snapshot.status == "sealed"
+            assert snapshot.archiveresult_set.count() == 1
+            assert snapshot.output_size == len(payload)
+            assert (snapshot.output_dir / "singlefile.html").read_bytes() == payload
+            assert (snapshot.output_dir / "index.json").read_text() == original_json
+            if result_id is not None:
+                assert result.pk == result_id
+            result_id = result.pk
+
+
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)

archivebox/tests/test_config_DELETE_AFTER.py

@@ -160,6 +160,10 @@ print(json.dumps({{
 
 
 def test_delete_after_real_add_page_and_rest_create_paths(client):
+    from archivebox.personas.models import Persona
+
+    # Match init/server startup; visiting /add/ must not create persona records.
+    Persona.get_or_create_default()
     User = get_user_model()
     admin_user = User.objects.create_superuser(
         username="retentionadmin",

archivebox/tests/test_config_MAX_limits.py

@@ -203,6 +203,7 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
 
     Snapshot.objects.filter(pk=root.pk).update(output_size=32)
 
+    root_output_dir = str(root.output_dir)
     bus = create_bus(name=f"test_snapshot_limit_cancel_{str(crawl.id).replace('-', '_')}")
     SnapshotService(bus, crawl_id=str(crawl.id))
     try:
@@ -212,14 +213,14 @@ def test_seal_snapshot_cancels_queued_descendants_after_crawl_max_size():
                 SnapshotEvent(
                     url=root.url,
                     snapshot_id=str(root.id),
-                    output_dir=str(root.output_dir),
+                    output_dir=root_output_dir,
                 ),
             )
             await snapshot_event.now()
             completed_event = SnapshotCompletedEvent(
                 url=root.url,
                 snapshot_id=str(root.id),
-                output_dir=str(root.output_dir),
+                output_dir=root_output_dir,
             )
             completed_event.event_parent_id = snapshot_event.event_id
             await bus.emit(completed_event).now()

archivebox/tests/test_config_ONLY_NEW.py

@@ -93,4 +93,6 @@ def test_create_discovered_snapshots_respects_only_new_exact_url_matches(admin_u
         crawl_link = (
             Path(crawl.output_dir) / CONSTANTS.SNAPSHOTS_DIR_NAME / snapshot.extract_domain_from_url(snapshot.url) / str(snapshot.id)
         )
-        assert crawl_link.is_symlink()
+        # Queued metadata stays in the DB; the normal runner creates links.
+        assert not crawl_link.is_symlink()
+        assert not snapshot.output_dir.exists()

archivebox/tests/test_core_config.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 from pydantic import ValidationError
@@ -95,3 +96,40 @@ def test_resolving_scoped_config_does_not_mutate_process_environment(tmp_path):
             os.environ.pop("ABXPKG_LIB_DIR", None)
         else:
             os.environ["ABXPKG_LIB_DIR"] = previous_lib_dir
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE", "WGET_COOKIES_FILE", "YTDLP_COOKIES_FILE", "GALLERYDL_COOKIES_FILE"])
+def test_cookie_paths_remain_collection_relative_after_runtime_overlays(key, tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: "auth/cookies.txt"})
+    assert str(config[key]) == str(CONSTANTS.DATA_DIR / "auth/cookies.txt")
+    runtime = config.for_crawl_runtime(runtime_overrides={key: "other/cookies.txt"})
+    assert runtime[key] == str(CONSTANTS.DATA_DIR / "other/cookies.txt")
+    absolute = tmp_path / "absolute-cookies.txt"
+    config = ArchiveBoxConfig(**{key: str(absolute)})
+    assert str(config[key]) == str(absolute)
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE"])
+@pytest.mark.parametrize("value", ["", "  ", None])
+def test_empty_cookie_path_disables_import(key, value):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: value})
+    assert config[key] is None
+    assert key not in config.for_crawl_runtime()
+
+
+def test_cookie_runtime_path_preserves_configured_symlink(tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    export = tmp_path / "export.txt"
+    export.write_text("# Netscape HTTP Cookie File\n")
+    configured = tmp_path / "cookies.txt"
+    configured.symlink_to(export)
+    config = ArchiveBoxConfig(COOKIES_FILE=configured)
+    assert str(config.COOKIES_FILE) == str(configured)
+    runtime = config.for_crawl_runtime()
+    assert runtime["COOKIES_FILE"] == str(configured)
+    assert Path(runtime["COOKIES_FILE"]).read_bytes() == export.read_bytes()

archivebox/tests/test_crawl_history.py

@@ -0,0 +1,111 @@
+"""Completed captures belong in the database, not a crawl-long event history."""
+
+import asyncio
+import gc
+import json
+import threading
+import weakref
+from collections import Counter
+
+import pytest
+
+from abx_dl.events import ProcessCompletedEvent, SnapshotCompletedEvent, SnapshotEvent
+from werkzeug import Response
+from archivebox.base_models.models import get_or_create_system_user_pk
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.services.runner import CrawlRunner, run_install
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("timeout_page", [False, True])
+def test_completed_snapshots_expire_after_database_projection(tmp_path, httpserver, hermetic_lib_dir, timeout_page):
+    urls = []
+    response_started = threading.Event()
+    release_response = threading.Event()
+
+    def stalled_response(_request):
+        def body():
+            response_started.set()
+            yield b"<html><head><title>Saved page 3</title></head><body>"
+            release_response.wait()
+            yield b"Completed response after the browser timed out.</body></html>"
+
+        return Response(body(), content_type="text/html")
+
+    httpserver.expect_request("/favicon.ico").respond_with_data("", status=204)
+    for index in range(8):
+        path = f"/page/{index}"
+        if timeout_page and index == 3:
+            httpserver.expect_request(path).respond_with_handler(stalled_response)
+            urls.append(httpserver.url_for(path))
+            continue
+        httpserver.expect_request(path).respond_with_data(
+            f"<html><head><title>Saved page {index}</title></head><body>"
+            "This real page is part of a multi-snapshot crawl whose completed captures "
+            "must remain available in the database after their events expire.</body></html>",
+            content_type="text/html",
+        )
+        urls.append(httpserver.url_for(path))
+    run_install(plugin_names=["title"])
+    crawl = Crawl.objects.create(
+        urls="\n".join(urls),
+        config={
+            "ABXPKG_LIB_DIR": str(hermetic_lib_dir),
+            "PLUGINS": "title",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": 1,
+            "CHROME_PAGELOAD_TIMEOUT": 5,
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
+    runner.bus.event_history.max_history_size = 120
+    sizes = []
+    references = []
+    counts = Counter()
+    retired_ids = set()
+    retained_completed = []
+
+    def count_event(event):
+        counts[event.event_type] += 1
+
+    def observe_snapshot(event: SnapshotEvent):
+        sizes.append(sum(isinstance(previous, SnapshotEvent) for previous in runner.bus.event_history.values()))
+        retained_completed.append(len(retired_ids.intersection(runner.bus.event_history)))
+        references.append(weakref.ref(event))
+
+    async def observe_completed(event: SnapshotCompletedEvent):
+        root = await runner.bus.find(SnapshotEvent, snapshot_id=event.snapshot_id, past=True, future=False)
+        assert root is not None
+        retired_ids.add(root.event_id)
+        retired_ids.update(item.event_id for item in await runner.bus.filter("*", child_of=root, past=True, future=False))
+
+    def release_timed_out_response(event: ProcessCompletedEvent):
+        if response_started.is_set() and event.hook_name == "on_Snapshot__30_chrome_navigate" and event.status == "failed":
+            release_response.set()
+
+    runner.bus.on("*", count_event)
+    runner.bus.on(SnapshotEvent, observe_snapshot)
+    runner.bus.on(SnapshotCompletedEvent, observe_completed)
+    runner.bus.on(ProcessCompletedEvent, release_timed_out_response)
+    try:
+        asyncio.run(runner.run())
+    finally:
+        release_response.set()
+    (tmp_path / "event-counts.json").write_text(json.dumps({"events": counts, "snapshots_in_history": sizes}))
+    snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("url"))
+    assert len(snapshots) == len(urls)
+    assert all(snapshot.status == "sealed" for snapshot in snapshots)
+    for index, snapshot in enumerate(snapshots):
+        if timeout_page and index == 3:
+            assert response_started.is_set()
+            result = ArchiveResult.objects.get(snapshot=snapshot, hook_name="on_Snapshot__30_chrome_navigate", status="failed")
+            assert "timeout" in result.output_str.lower()
+            continue
+        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", status="succeeded")
+        assert result.output_str == f"Saved page {index}"
+        assert (snapshot.output_dir / "title" / "title.txt").read_text().strip() == f"Saved page {index}"
+    assert sizes == [1] * len(urls), sizes
+    assert retained_completed == [0] * len(urls), retained_completed
+    gc.collect()
+    assert all(reference() is None for reference in references)

archivebox/tests/test_crawl_runner.py

@@ -71,6 +71,7 @@ def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus():
         crawl=crawl,
         status=Snapshot.StatusChoices.STARTED,
     )
+    crawl_output_dir = str(crawl.output_dir)
     runner = CrawlRunner(crawl)
 
     async def run() -> CrawlAbortEvent | None:
@@ -90,7 +91,7 @@ def test_cancelled_crawl_projection_emits_abort_event_from_runner_bus():
             CrawlEvent(
                 url=snapshot.url,
                 snapshot_id=str(snapshot.id),
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             ),
         ).now()
         await runner.bus.wait_until_idle()
@@ -323,6 +324,7 @@ def test_crawl_start_event_keeps_retry_at_lease():
         status=Snapshot.StatusChoices.QUEUED,
         retry_at=before,
     )
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name="test_crawl_start_event_keeps_retry_at_lease")
     CrawlService(bus, crawl_id=str(crawl.id))
 
@@ -332,7 +334,7 @@ def test_crawl_start_event_keeps_retry_at_lease():
                 CrawlStartEvent(
                     url=snapshot.url,
                     snapshot_id=str(snapshot.id),
-                    output_dir=str(crawl.output_dir),
+                    output_dir=crawl_output_dir,
                 ),
             ).now()
             await bus.wait_until_idle()
@@ -979,6 +981,7 @@ def test_crawl_completed_event_requeues_active_snapshots():
         retry_at=None,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_completed_active_snapshots_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -988,7 +991,7 @@ def test_crawl_completed_event_requeues_active_snapshots():
             event = CrawlCompletedEvent(
                 url="https://example.com",
                 snapshot_id="",
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1022,6 +1025,7 @@ def test_crawl_start_event_does_not_resurrect_cancelled_crawl():
         retry_at=now,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_start_cancelled_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -1031,7 +1035,7 @@ def test_crawl_start_event_does_not_resurrect_cancelled_crawl():
             event = CrawlStartEvent(
                 url="https://example.com",
                 snapshot_id="",
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1069,6 +1073,7 @@ def test_crawl_cleanup_event_requeues_unfinished_crawl():
         retry_at=None,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_cleanup_requeues_unfinished_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -1078,7 +1083,7 @@ def test_crawl_cleanup_event_requeues_unfinished_crawl():
             event = CrawlCleanupEvent(
                 url="https://example.com",
                 snapshot_id=str(snapshot.id),
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1117,6 +1122,7 @@ def test_crawl_completed_event_seals_finished_crawl():
         retry_at=None,
     )
 
+    crawl_output_dir = str(crawl.output_dir)
     bus = create_bus(name=f"test_crawl_completed_finished_crawl_{str(crawl.id).replace('-', '_')}")
     service = CrawlService(bus, crawl_id=str(crawl.id))
     assert service is not None
@@ -1126,7 +1132,7 @@ def test_crawl_completed_event_seals_finished_crawl():
             event = CrawlCompletedEvent(
                 url="https://example.com",
                 snapshot_id=str(snapshot.id),
-                output_dir=str(crawl.output_dir),
+                output_dir=crawl_output_dir,
             )
             emitted = bus.emit(event)
             await emitted.wait()
@@ -1165,6 +1171,7 @@ def test_snapshot_completed_event_bus_defers_finished_crawl_seal():
         retry_at=None,
     )
 
+    snapshot_output_dir = str(snapshot.output_dir)
     bus = create_bus(name=f"test_snapshot_completed_bus_finished_crawl_{str(crawl.id).replace('-', '_')}")
     SnapshotService(bus, crawl_id=str(crawl.id))
     try:
@@ -1174,14 +1181,14 @@ def test_snapshot_completed_event_bus_defers_finished_crawl_seal():
                 SnapshotEvent(
                     url="https://example.com",
                     snapshot_id=str(snapshot.id),
-                    output_dir=str(snapshot.output_dir),
+                    output_dir=snapshot_output_dir,
                 ),
             )
             await snapshot_event.now()
             completed_event = SnapshotCompletedEvent(
                 url="https://example.com",
                 snapshot_id=str(snapshot.id),
-                output_dir=str(snapshot.output_dir),
+                output_dir=snapshot_output_dir,
             )
             completed_event.event_parent_id = snapshot_event.event_id
             await bus.emit(completed_event).now()
@@ -1221,24 +1228,25 @@ def test_delayed_snapshot_completion_cannot_seal_new_run():
         retry_at=old_retry_at,
     )
 
+    snapshot_output_dir = str(snapshot.output_dir)
     bus = create_bus(name=f"test_delayed_snapshot_completion_{str(crawl.id).replace('-', '_')}")
     SnapshotService(bus, crawl_id=str(crawl.id))
     try:
 
         async def emit_runs() -> None:
             old_event = bus.emit(
-                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir)),
+                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=snapshot_output_dir),
             )
             await old_event.now()
             await sync_to_async(Snapshot.objects.filter(pk=snapshot.pk).update, thread_sensitive=True)(retry_at=new_retry_at)
             new_event = bus.emit(
-                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=str(snapshot.output_dir)),
+                SnapshotEvent(url=snapshot.url, snapshot_id=str(snapshot.id), output_dir=snapshot_output_dir),
             )
             await new_event.now()
             completed_event = SnapshotCompletedEvent(
                 url=snapshot.url,
                 snapshot_id=str(snapshot.id),
-                output_dir=str(snapshot.output_dir),
+                output_dir=snapshot_output_dir,
             )
             completed_event.event_parent_id = old_event.event_id
             await bus.emit(completed_event).now()

archivebox/tests/test_hook_result_lifecycle.py

@@ -70,6 +70,11 @@ def test_cancelled_hook_discards_result_without_deleting_partial_files(snapshot,
 
 @pytest.mark.parametrize("stop_mode", ["crash", "cancel"])
 def test_stopped_hook_reconciles_its_earlier_success(initialized_archive, stop_mode):
+    env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True")
+    # The deadline below measures a running hook's lifecycle, not a cold Chrome
+    # download/install. Install through the public CLI before starting that clock.
+    installed = run_archivebox_cmd(["install", "chrome"], cwd=initialized_archive, env=env, timeout=600)
+    assert installed.returncode == 0, installed.stdout + installed.stderr
     output = initialized_archive / "crash-check.log"
     cli = None
     with output.open("w") as log:
@@ -77,7 +82,7 @@ def test_stopped_hook_reconciles_its_earlier_success(initialized_archive, stop_m
             cli = run_archivebox_cmd(
                 ["add", "--plugins=chrome", "https://example.com"],
                 cwd=initialized_archive,
-                env=cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True"),
+                env=env,
                 stdin=subprocess.DEVNULL,
                 stdout=log,
                 stderr=log,

archivebox/tests/test_issue_1896.py

@@ -0,0 +1,193 @@
+"""Regression coverage for collection cookie paths and config edits on upgrade."""
+
+import json
+import sqlite3
+from contextlib import closing
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+from archivebox.config.configset import read_ini_config
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+@pytest.mark.parametrize("persona_name", ["Default", "default"])
+def test_add_reuses_persona_ignoring_case(initialized_archive, httpserver, persona_name):
+    """Either spelling reuses the same authenticated persona without moving files."""
+    run_archivebox_cmd(["persona", "create", "Default", "default"], cwd=initialized_archive, check=True)
+    persona_dir = initialized_archive / "personas" / "Default"
+    (persona_dir / "auth.json").touch()
+    (persona_dir / "cookies.txt").write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+    )
+    original_inode = (persona_dir / "cookies.txt").stat().st_ino
+    httpserver.expect_request("/").respond_with_data(
+        "<html><head><title>Existing persona capture</title></head><body>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script>'
+        "</body></html>",
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--persona", persona_name, "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert "issue1896=imported" in outputs[0].read_text()
+    assert (persona_dir / "cookies.txt").stat().st_ino == original_inode
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona ORDER BY name").fetchall() == [("Default",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_envs = db.execute("SELECT env FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_envs
+        assert all(json.loads(row[0])["ACTIVE_PERSONA"] == "Default" for row in hook_envs)
+
+
+@pytest.mark.parametrize("contents,explicit", [("", True), ("not JSON", True), ("not JSON", False)])
+def test_add_rejects_invalid_persona_auth_with_file_path(initialized_archive, httpserver, contents, explicit):
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.write_text(contents)
+    if explicit:
+        run_archivebox_cmd(
+            ["config", "--set", "AUTH_STORAGE_FILE=personas/Default/auth.json"],
+            cwd=initialized_archive,
+            check=True,
+        )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 1, result.stdout + result.stderr
+    logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    assert logs
+    assert f"Invalid JSON cookie export: {auth_file}" in "\n".join(log.read_text() for log in logs)
+    assert not list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert auth_file.read_text() == contents
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("isolation", ["crawl", "snapshot"])
+def test_add_with_empty_autodiscovered_persona_auth(initialized_archive, httpserver, cookie_file_exists, isolation):
+    """An empty auth placeholder must not block a crawl or hide Netscape cookies."""
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.touch()
+    cookies_file = persona_dir / "cookies.txt"
+    cookies_file.write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n" if cookie_file_exists else "",
+    )
+    httpserver.expect_request("/").respond_with_data(
+        '<html><title>Empty persona auth</title><body><output id="cookies"></output>'
+        '<script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false", "CHROME_ISOLATION": isolation},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert ("issue1896=imported" in outputs[0].read_text()) == cookie_file_exists
+    assert auth_file.read_bytes() == b""
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT cmd, env, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for cmd, env, exit_code in hook_rows:
+            hook_env = json.loads(env)
+            assert not hook_env.get("AUTH_STORAGE_FILE")
+            expected_cookies = cookies_file if isolation == "crawl" else Path(hook_env["PERSONAS_DIR"]) / "Default" / "cookies.txt"
+            assert hook_env["COOKIES_FILE"] == str(expected_cookies)
+            hook_name = Path(json.loads(cmd)[0]).name
+            # The crawl wait hook explicitly skips when each snapshot owns Chrome.
+            assert exit_code == (10 if isolation == "snapshot" and hook_name == "on_CrawlSetup__91_chrome_wait.js" else 0)
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("absolute_path", [False, True], ids=["collection-relative", "absolute"])
+def test_add_with_collection_relative_cookies(initialized_archive, httpserver, cookie_file_exists, absolute_path):
+    """Real hooks resolve relative paths against DATA_DIR and preserve absolute paths."""
+    cookies_file = (
+        initialized_archive.parent / f"{initialized_archive.name}-cookies.txt" if absolute_path else initialized_archive / "cookies.txt"
+    )
+    configured_path = str(cookies_file) if absolute_path else "cookies.txt"
+    if cookie_file_exists:
+        cookies_file.write_text(
+            "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+        )
+    httpserver.expect_request("/").respond_with_data(
+        "<html><title>Issue 1896</title><body><p>Collection cookie import regression: "
+        "this page must still be archived when the optional cookie file is absent.</p>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    run_archivebox_cmd(
+        ["config", "--set", f"COOKIES_FILE={configured_path}"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    html = outputs[0].read_text()
+    assert ("issue1896=imported" in html) == cookie_file_exists
+    launch_logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    logs = "\n".join(log.read_text() for log in launch_logs)
+    if cookie_file_exists:
+        assert f"from {cookies_file}" in logs
+    else:
+        assert "Cookies file not found" in logs
+        assert "continuing without cookie import" in logs
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT env, status, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for env, status, exit_code in hook_rows:
+            assert json.loads(env)["COOKIES_FILE"] == str(cookies_file)
+            assert (status, exit_code) == ("exited", 0)
+
+
+@pytest.mark.parametrize("enabled_key", ["USE_CHROME", "CHROME_ENABLED"])
+@pytest.mark.parametrize("startup_save", ["binary_sanitation", "host_refresh"])
+def test_init_preserves_edited_use_chrome_before_startup_save(initialized_archive, enabled_key, startup_save):
+    """Startup saves must not overwrite a newer file before config sync."""
+    config_path = initialized_archive / "ArchiveBox.conf"
+    config_path.write_text(config_path.read_text() + f"\n[SERVER_CONFIG]\n{enabled_key} = True\n")
+    run_archivebox_cmd(
+        ["config", "--set", f"WGET_BINARY={initialized_archive / 'removed-wget'}"]
+        if startup_save == "binary_sanitation"
+        else ["config", "--set", "TIMEOUT=37"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    if startup_save == "host_refresh":
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            db.execute("UPDATE machine_machine SET modified_at=?", [(datetime.now(timezone.utc) - timedelta(days=8)).isoformat()])
+            db.commit()
+    config_path.write_text(config_path.read_text().replace(f"{enabled_key} = True", f"{enabled_key} = False"))
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    run_archivebox_cmd(["init"], cwd=initialized_archive, timeout=120, check=True)
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    result = run_archivebox_cmd(["config", "--get", "CHROME_ENABLED"], cwd=initialized_archive)
+    assert result.returncode == 0, result.stderr
+    assert "CHROME_ENABLED = false" in result.stdout

archivebox/tests/test_machine_models.py

@@ -1005,3 +1005,58 @@ class TestProcessClassMethods:
 
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])
+
+
+@pytest.mark.parametrize("cleanup", ["cleanup_stale_running", "cleanup_orphaned_workers"])
+def test_process_cleanup_releases_reads_before_process_checks(cleanup, tmp_path):
+    """Another real connection can write while recovery inspects live processes."""
+    import sqlite3
+    from django.db import connection
+
+    from archivebox.tests.conftest import init_archive
+    from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+    init_archive(tmp_path)
+    with use_archivebox_db(tmp_path):
+        machine = Machine.current()
+        processes = [
+            Process(
+                machine=machine,
+                process_type=Process.TypeChoices.HOOK,
+                status=Process.StatusChoices.RUNNING,
+                pid=os.getpid(),
+                started_at=_current_process_started_at(),
+            )
+            for _ in range(105)
+        ]
+        Process.objects.bulk_create(processes)
+        with connection.cursor() as cursor:
+            cursor.execute("PRAGMA journal_mode=DELETE")
+        blocked = []
+        checks = []
+        writer = sqlite3.connect(connection.settings_dict["NAME"], timeout=0.01, uri=True)
+
+        def observe(frame, event, arg):
+            if (
+                event == "call"
+                and frame.f_code.co_name == ("poll" if cleanup == "cleanup_stale_running" else "is_running")
+                and isinstance(frame.f_locals.get("self"), Process)
+            ):
+                checks.append(1)
+                try:
+                    writer.execute("UPDATE machine_machine SET hostname=hostname WHERE id=?", [machine.pk.hex])
+                    writer.commit()
+                except sqlite3.OperationalError as error:
+                    writer.rollback()
+                    blocked.append(str(error))
+
+        previous = sys.getprofile()
+        sys.setprofile(observe)
+        try:
+            getattr(Process, cleanup)()
+        finally:
+            sys.setprofile(previous)
+            writer.close()
+        assert len(checks) >= len(processes)
+        assert blocked == []
+        assert Process.objects.filter(pk__in=[p.pk for p in processes], status=Process.StatusChoices.RUNNING).count() == len(processes)

archivebox/tests/test_opencode_agent_2.py

@@ -63,7 +63,7 @@ def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode
     frame_path = runtime._project_route(live_opencode.config.data_dir)
 
     assert response.status_code == 200
-    assert f'<iframe src="{frame_path}"'.encode() in response.content
+    assert f'<iframe data-src="{frame_path}"'.encode() in response.content
     assert b'id="opencode-agent-welcome"' in response.content
     assert b'id="header"' in response.content
     assert b'id="progress-monitor"' in response.content

archivebox/tests/test_opencode_agent_3.py

@@ -44,7 +44,7 @@ def test_opencode_cold_agent_wrapper_returns_before_server_starts(admin_client,
     assert time.monotonic() - started < 3
     assert not runtime._owned_process_running()
     assert b'id="opencode-agent-welcome"' in response.content
-    assert f'<iframe src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
+    assert f'<iframe data-src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
 
 
 def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):

archivebox/tests/test_opencode_browser_1.py

@@ -92,8 +92,10 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
   try {
     const page = await browser.newPage();
     const escapedRequests = [];
+    const agentRequests = [];
     page.on('request', request => {
       const url = new URL(request.url());
+      if (url.pathname.startsWith('/admin/agent/opencode/')) agentRequests.push(url.pathname);
       if (request.frame()?.parentFrame() && url.origin === config.url &&
           !url.pathname.startsWith('/admin/agent/opencode/')) {
         escapedRequests.push(request.method() + ' ' + url.pathname);
@@ -105,8 +107,11 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
     await page.locator('#login-form input[name="password"]').fill('test-password');
     await Promise.all([page.waitForNavigation({waitUntil: 'domcontentloaded'}),
       page.locator('#login-form input[type="submit"]').click()]);
-    assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'domcontentloaded'})).status(), 200);
-    await page.locator('#opencode-agent-welcome-dismiss').click();
+    assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'load'})).status(), 200);
+    // The welcome dialog must not start a hidden OpenCode app: its cold startup
+    // and asset load compete with rendering the very dialog the user must read.
+    assert.deepEqual(agentRequests, [], 'OpenCode started before the user dismissed its welcome dialog');
+    await page.locator('::-p-aria(Start using Agent[role="button"])').click();
     const frame = await (await page.waitForSelector('iframe')).contentFrame();
     await frame.waitForSelector('a[href*="/session"]');
     const links = await frame.$$eval('a[href*="/session"]', nodes => nodes.map(node => node.getAttribute('href')));
@@ -264,6 +269,12 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
     assert.ok(terminal.includes('ABX_TERMINAL_OK'), terminal);
     assert.deepEqual(escapedRequests, [], 'OpenCode requests must stay inside its proxy mount');
     assert.equal((await page.goto(config.url + '/add/', {waitUntil: 'domcontentloaded'})).status(), 200);
+    // A real click persists the dismissal: returning users load the Agent
+    // directly, without another click or seeded browser preferences.
+    assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'load'})).status(), 200);
+    await page.waitForSelector('::-p-aria(Start using Agent[role="button"])', {hidden: true});
+    const returningFrame = await (await page.waitForSelector('iframe[title="OpenCode Agent"]')).contentFrame();
+    await returningFrame.waitForSelector('::-p-aria(New session[role="button"])');
     console.log('AGENT_NAVIGATION_OK');
   } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exitCode = 1; });

archivebox/tests/test_persona_casing.py

@@ -0,0 +1,237 @@
+"""Persona names have the same identity on case-sensitive and insensitive disks."""
+
+import json
+import sqlite3
+import subprocess
+import sys
+from contextlib import closing
+
+import pytest
+from django.urls import reverse
+
+from archivebox.personas.models import Persona
+from archivebox.tests.conftest import ADMIN_TEST_HOST, api_client_request, get_free_port, run_archivebox_cmd, stop_server
+
+
+def test_cli_create_reuses_case_insensitive_persona(initialized_archive):
+    first = run_archivebox_cmd(["persona", "create", "--permissions=private", "personal"], cwd=initialized_archive, check=True)
+    second = run_archivebox_cmd(["persona", "create", "--permissions=public", "PERSONAL"], cwd=initialized_archive, check=True)
+    assert json.loads(second.stdout)["id"] == json.loads(first.stdout)["id"]
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        rows = db.execute("SELECT name, config FROM personas_persona WHERE name != 'Default'").fetchall()
+    assert len(rows) == 1
+    assert rows[0][0] == "personal"
+    assert json.loads(rows[0][1])["PERMISSIONS"] == "private"
+
+
+@pytest.mark.django_db
+def test_add_form_reuses_existing_lowercase_default(admin_client, admin_user):
+    original = Persona.objects.create(name="default", created_by=admin_user)
+    response = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    assert list(Persona.objects.values_list("id", "name")) == [(original.id, "default")]
+
+
+@pytest.mark.django_db
+def test_add_form_does_not_create_personas(admin_client):
+    assert not Persona.objects.exists()
+    response = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    assert not Persona.objects.exists()
+
+
+def test_init_creates_default_persona(initialized_archive):
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    assert (initialized_archive / "personas" / "Default" / "chrome_profile").is_dir()
+
+
+def test_server_initializes_default_persona_for_existing_collections(initialized_archive):
+    from archivebox.tests.conftest import cli_env, get_free_port, start_archivebox_server, stop_server
+
+    # Pre-fix collections may have no persona row until the first visit to /add/.
+    run_archivebox_cmd(
+        ["manage", "shell", "--no-imports", "-c", "from archivebox.personas.models import Persona; Persona.objects.all().delete()"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    port = get_free_port()
+    try:
+        start_archivebox_server(initialized_archive, port=port, env=cli_env(port=port, server=True))
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    finally:
+        stop_server(initialized_archive)
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_delete_preserves_files_and_recreation_adopts_directory_case(admin_client, admin_user):
+    persona = Persona.objects.create(name="personal", created_by=admin_user)
+    persona.ensure_dirs()
+    profile = persona.path / "chrome_profile" / "Preferences"
+    profile.write_text('{"profile":{"name":"Preserve me"}}')
+    cookies = persona.path / "cookies.txt"
+    cookies.write_text("# Netscape HTTP Cookie File\n")
+    before = (profile.stat().st_ino, cookies.stat().st_ino)
+    response = admin_client.post(reverse("admin:personas_persona_delete", args=[persona.pk]), {"post": "yes"}, HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 302
+    assert not Persona.objects.filter(pk=persona.pk).exists()
+    assert profile.read_text() == '{"profile":{"name":"Preserve me"}}'
+    response = admin_client.post(
+        reverse("admin:personas_persona_add"),
+        {
+            "name": "PERSONAL",
+            "created_by": str(admin_user.pk),
+            "permissions": "private",
+            "config": "{}",
+            "import_mode": "none",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302, response.content
+    recreated = Persona.objects.get()
+    assert recreated.name == "personal"
+    assert recreated.path == profile.parent.parent
+    assert (profile.stat().st_ino, cookies.stat().st_ino) == before
+
+
+@pytest.mark.django_db(transaction=True)
+def test_extension_sync_reuses_persona_case(client, api_headers):
+    original = Persona.objects.create(name="personal", config={"TIMEOUT": 37})
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": "case-test", "name": "PERSONAL", "settings": {}},
+        headers=api_headers,
+    )
+    assert response.status_code == 200, response.content
+    assert response.json()["created"] is False
+    assert response.json()["persona"]["id"] == str(original.pk)
+    assert Persona.objects.count() == 1
+    original.refresh_from_db()
+    assert original.config["TIMEOUT"] == 37
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_duplicate_name_identifies_existing_record_and_directory(admin_client, admin_user):
+    original = Persona.objects.create(name="personal", created_by=admin_user)
+    original.ensure_dirs()
+    response = admin_client.post(
+        reverse("admin:personas_persona_add"),
+        {
+            "name": "PERSONAL",
+            "created_by": str(admin_user.pk),
+            "permissions": "private",
+            "config": "{}",
+            "import_mode": "none",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 200
+    assert str(original.pk).encode() in response.content
+    assert str(original.path).encode() in response.content
+    assert list(Persona.objects.values_list("id", "name")) == [(original.id, "personal")]
+
+
+@pytest.mark.django_db(transaction=True)
+def test_admin_case_only_edit_preserves_profile_directory(admin_client, admin_user):
+    persona = Persona.objects.create(name="personal", created_by=admin_user)
+    persona.ensure_dirs()
+    profile = persona.path / "chrome_profile" / "Preferences"
+    profile.write_text('{"profile":{"name":"Keep this profile"}}')
+    inode = profile.stat().st_ino
+    response = admin_client.post(
+        reverse("admin:personas_persona_change", args=[persona.pk]),
+        {"name": "PERSONAL", "created_by": str(admin_user.pk), "permissions": "private", "config": "{}", "_save": "Save"},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302, response.content
+    persona.refresh_from_db()
+    assert persona.name == "personal"
+    assert profile.stat().st_ino == inode
+    assert profile.read_text() == '{"profile":{"name":"Keep this profile"}}'
+
+
+@pytest.mark.django_db(transaction=True)
+def test_extension_sync_rejects_legacy_case_conflicts_before_writing(client, api_headers):
+    # Real rows representing collections created before case-insensitive matching.
+    Persona.objects.bulk_create([Persona(name="personal"), Persona(name="PERSONAL")])
+    response = api_client_request(
+        client,
+        "post",
+        "/api/v1/personas/sync",
+        payload={"extension_persona_id": "case-test-conflict", "name": "Personal", "settings": {}, "cookies_txt": "must not be written"},
+        headers=api_headers,
+    )
+    assert response.status_code == 409, response.content
+    assert "rename" in response.content.decode().lower()
+    assert "manually" in response.content.decode().lower()
+    assert Persona.objects.count() == 2
+    assert all(str(persona.pk).encode() in response.content for persona in Persona.objects.all())
+    assert all(not (persona.path / "cookies.txt").exists() for persona in Persona.objects.all())
+
+
+@pytest.fixture
+def case_sensitive_collection(tmp_path):
+    """Exercise actual Linux-style directory collisions, including on macOS."""
+    mount = tmp_path / "case-sensitive"
+    mount.mkdir()
+    if sys.platform == "darwin":
+        image = tmp_path / "personas.sparseimage"
+        subprocess.run(
+            ["hdiutil", "create", "-size", "128m", "-type", "SPARSE", "-fs", "Case-sensitive APFS", "-volname", "PersonaTest", str(image)],
+            check=True,
+            capture_output=True,
+        )
+        subprocess.run(["hdiutil", "attach", "-nobrowse", "-mountpoint", str(mount), str(image)], check=True, capture_output=True)
+    try:
+        yield mount
+    finally:
+        if sys.platform == "darwin":
+            subprocess.run(["hdiutil", "detach", str(mount)], check=True, capture_output=True)
+
+
+def test_cli_rejects_case_conflicting_directories_without_mutating(case_sensitive_collection):
+    collection = case_sensitive_collection
+    run_archivebox_cmd(["init", "--quick"], cwd=collection, check=True)
+    for name in ("Default", "default"):
+        profile = collection / "personas" / name / "chrome_profile"
+        profile.mkdir(parents=True, exist_ok=True)
+        (profile / "Preferences").write_text(name)
+    commands = (["persona", "create", "Default"], ["init", "--quick"], ["server", "--daemonize", f"127.0.0.1:{get_free_port()}"])
+    try:
+        for command in commands:
+            result = run_archivebox_cmd(command, cwd=collection)
+            assert result.returncode != 0
+            assert "rename" in result.stderr.lower()
+            assert "manually" in result.stderr.lower()
+            assert all(str(collection / "personas" / name) in result.stderr for name in ("Default", "default"))
+    finally:
+        stop_server(collection)
+    with closing(sqlite3.connect(collection / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona").fetchall() == [("Default",)]
+    for name in ("Default", "default"):
+        assert (collection / "personas" / name / "chrome_profile" / "Preferences").read_text() == name
+
+
+def test_cli_adopts_existing_directory_spelling_on_case_sensitive_disk(case_sensitive_collection):
+    collection = case_sensitive_collection
+    run_archivebox_cmd(["init", "--quick"], cwd=collection, check=True)
+    profile = collection / "personas" / "personal" / "chrome_profile" / "Preferences"
+    profile.parent.mkdir(parents=True)
+    profile.write_text('{"profile":{"name":"Keep me"}}')
+    inode = profile.stat().st_ino
+    run_archivebox_cmd(["persona", "create", "PERSONAL"], cwd=collection, check=True)
+    with closing(sqlite3.connect(collection / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona ORDER BY name").fetchall() == [("Default",), ("personal",)]
+        # A legacy database can disagree with an existing directory's spelling.
+        db.execute("UPDATE personas_persona SET name='PERSONAL' WHERE name='personal'")
+        db.commit()
+    result = run_archivebox_cmd(["persona", "create", "Personal"], cwd=collection, check=True)
+    assert json.loads(result.stdout)["name"] == "personal"
+    assert not (collection / "personas" / "PERSONAL").exists()
+    assert profile.stat().st_ino == inode
+    assert profile.read_text() == '{"profile":{"name":"Keep me"}}'

archivebox/tests/test_release_workflow.py

@@ -185,13 +185,22 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
 
     assert all(step.get("name") != "Verify published PyPI package installs and runs" for step in python_release["steps"])
     ci = yaml.safe_load(CI_WORKFLOW.read_text())
-    screenshots = yaml.safe_load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text())
+    screenshots = yaml.load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text(), Loader=yaml.BaseLoader)
     prepare_path = ci["jobs"]["prepare"]["uses"]
-    assert screenshots["jobs"]["prepare"]["uses"] == prepare_path
     prepare = yaml.safe_load((REPO_ROOT / prepare_path).read_text())
     assert prepare["jobs"]["prepare"]["steps"][-1]["uses"] == "ArchiveBox/monorepo/.github/actions/prepare-release-version@main"
-    assert screenshots["jobs"]["deploy"]["needs"] == "prepare"
-    assert screenshots["jobs"]["deploy"]["if"] == "needs.prepare.outputs.run_jobs == 'true'"
+    assert set(screenshots["on"]) == {"schedule", "workflow_dispatch"}
+    _, hours, *days = screenshots["on"]["schedule"][0]["cron"].split()
+    assert days == ["*", "*", "*"]
+    assert len(hours.split(",")) == 2
+    first, second = map(int, hours.split(","))
+    assert second - first == 12
+    assert "prepare" not in screenshots["jobs"]
+    assert screenshots["jobs"]["changes"]["steps"][0]["uses"] == "ArchiveBox/monorepo/.github/actions/changed-scheduled-inputs@main"
+    assert screenshots["jobs"]["deploy"]["needs"] == "changes"
+    assert screenshots["jobs"]["deploy"]["if"] == "needs.changes.outputs.changed == 'true'"
+    assert screenshots["jobs"]["complete"]["needs"] == ["changes", "deploy"]
+    assert screenshots["jobs"]["complete"]["name"] == "Scheduled inputs: ${{ needs.changes.outputs.key }}"
     for name, job in ci["jobs"].items():
         if name not in {"prepare", "required"}:
             assert job["needs"] == "prepare"

archivebox/tests/test_runner_deletion.py

@@ -0,0 +1,182 @@
+"""Exercise queued deletion in the existing runner with a real server."""
+
+import json
+import textwrap
+import time
+from pathlib import Path
+
+import pytest
+
+from archivebox.tests.conftest import (
+    cli_env,
+    get_free_port,
+    kill_processes_for_data_dir,
+    resolve_abxpkg_chrome_env,
+    run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    wait_for_log,
+)
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("restart", [False, True])
+def test_runner_retries_failed_deletion_without_blocking_captures(initialized_archive, recursive_test_site, restart):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+
+    port = get_free_port()
+    env = cli_env(
+        live=True,
+        server=True,
+        port=port,
+        PLUGINS="title",
+        SAVE_TITLE="True",
+        SEARCH_BACKEND_ENGINE="ripgrep",
+    )
+    # Measure captures while deletion retries, not a cold browser installation.
+    # Without setup, only the first case spends its capture deadline installing
+    # Chrome; later cases silently benefit from the shared binary cache.
+    env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env))
+    prepared = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            "import json; from archivebox.crawls.models import Crawl; "
+            "from archivebox.core.models import Snapshot, ArchiveResult; "
+            "from archivebox.base_models.models import get_or_create_system_user_pk; "
+            "c=Crawl.objects.create(urls='https://example.com/delete',created_by_id=get_or_create_system_user_pk(),status='sealed'); "
+            "s=Snapshot.objects.create(url='https://example.com/delete',crawl=c,status='deleting'); "
+            "p=s.output_dir/'wget'; p.mkdir(parents=True,exist_ok=True); "
+            "(p/'index.html').write_text('keep this until cleanup succeeds'); "
+            "ArchiveResult.objects.create(snapshot=s,plugin='wget',hook_name='on_Snapshot__06_wget',status='succeeded'); "
+            "p.chmod(0o500); print(json.dumps({'id':str(s.pk),'path':str(p)}))",
+        ],
+        cwd=initialized_archive,
+        env=env,
+    )
+    assert prepared.returncode == 0, prepared.stderr or prepared.stdout
+    fixture = json.loads(prepared.stdout.strip().splitlines()[-1])
+    protected = Path(fixture["path"])
+    worker_log = initialized_archive / "logs" / "worker_runner.log"
+    server = None
+    try:
+        server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-worker-server.log")
+        wait_for_log(worker_log, "PermissionError", timeout=30)
+        with use_archivebox_db(initialized_archive):
+            assert Process.objects.filter(worker_type="worker_runner", status="running").exists()
+            assert not Process.objects.filter(worker_type="worker_snapshot_delete").exists()
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+            assert ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert (protected / "index.html").is_file()
+        added = run_archivebox_cmd(
+            ["add", "--bg", "--plugins=title", recursive_test_site["root_url"]],
+            cwd=initialized_archive,
+            env=env,
+        )
+        assert added.returncode == 0, added.stderr or added.stdout
+        deadline = time.monotonic() + 60
+        with use_archivebox_db(initialized_archive):
+            while time.monotonic() < deadline:
+                captured = Snapshot.objects.filter(url=recursive_test_site["root_url"], status="sealed").first()
+                if captured is not None:
+                    break
+                assert server.poll() is None
+                time.sleep(0.1)
+            else:
+                pytest.fail((initialized_archive / "logs" / "worker_runner.log").read_text()[-12000:])
+            result = ArchiveResult.objects.get(snapshot=captured, plugin="title", status="succeeded")
+            assert result.output_files
+            assert captured.title == "Root"
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+        if restart:
+            stop_archivebox_process(server)
+            server = None
+        protected.chmod(0o700)
+        if restart:
+            server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-restarted-server.log")
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(initialized_archive):
+            while Snapshot.objects.filter(pk=fixture["id"]).exists() and time.monotonic() < deadline:
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=fixture["id"]).exists(), worker_log.read_text()
+            assert not ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert not protected.parent.exists()
+    finally:
+        if protected.exists():
+            protected.chmod(0o700)
+        if server is not None:
+            stop_archivebox_process(server)
+        kill_processes_for_data_dir(initialized_archive)
+
+
+def test_deletion_releases_database_before_filesystem_io(initialized_archive):
+    result = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            textwrap.dedent("""
+                import json
+                import sqlite3
+                import sys
+                from pathlib import Path
+                from django.db import connection
+                from archivebox.base_models.models import get_or_create_system_user_pk
+                from archivebox.core.models import ArchiveResult, Snapshot
+                from archivebox.crawls.models import Crawl
+                from archivebox.services.runner import run_pending_crawls
+
+                # DELETE mode exposes both read-cursor and transaction locks.
+                with connection.cursor() as cursor:
+                    cursor.execute('PRAGMA journal_mode=DELETE')
+                    assert cursor.fetchone()[0] == 'delete'
+                crawl = Crawl.objects.create(
+                    urls='https://example.com/delete',
+                    created_by_id=get_or_create_system_user_pk(), status='sealed',
+                )
+                paths = set()
+                snapshot_ids = []
+                for i in range(2):
+                    snapshot = Snapshot.objects.create(url=f'https://example.com/delete/{i}', crawl=crawl, status='deleting')
+                    path = snapshot.output_dir
+                    path.mkdir(parents=True, exist_ok=True)
+                    (path / 'saved.html').write_text('real saved output')
+                    ArchiveResult.objects.create(snapshot=snapshot, plugin='wget', status='succeeded')
+                    paths.add(str(path))
+                    snapshot_ids.append(snapshot.pk)
+                checked = set()
+
+                def inspect_filesystem_operation(event, args):
+                    if event != 'shutil.rmtree' or str(args[0]) not in paths:
+                        return
+                    assert not connection.in_atomic_block
+                    # Observe the real rmtree call without replacing it. An
+                    # independent writer must acquire its lock immediately.
+                    writer = sqlite3.connect(connection.settings_dict['NAME'], timeout=0, isolation_level=None)
+                    try:
+                        writer.execute('BEGIN EXCLUSIVE')
+                        writer.execute('UPDATE crawls_crawl SET notes=? WHERE id=?', ('concurrent write succeeded', crawl.pk.hex))
+                        writer.execute('COMMIT')
+                    finally:
+                        writer.close()
+                    checked.add(str(args[0]))
+
+                sys.addaudithook(inspect_filesystem_operation)
+                run_pending_crawls(maintenance_only=True)
+                assert checked == paths
+                assert not Snapshot.objects.filter(crawl=crawl).exists()
+                assert not ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids).exists()
+                assert all(not Path(path).exists() for path in paths)
+                crawl.refresh_from_db()
+                assert crawl.notes == 'concurrent write succeeded'
+                print(json.dumps({'verified_deletions': len(checked)}))
+            """),
+        ],
+        cwd=initialized_archive,
+        env=cli_env(disable_extractors=True),
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert json.loads(result.stdout.strip().splitlines()[-1]) == {"verified_deletions": 2}

archivebox/tests/test_server_security_browser.py

@@ -386,8 +386,11 @@ main().catch(async (error) => {
 
 
 @pytest.fixture
-def browser_runtime(initialized_archive: Path):
-    shared_lib = initialized_archive / "lib"
+def browser_runtime(cached_abxpkg_lib_dir):
+    # Security modes need isolated collections, but the browser installation is
+    # shared. Rebuilding it for every mode adds unrelated registry dependencies
+    # and discards the real binary cache already provided by the test helpers.
+    shared_lib = cached_abxpkg_lib_dir
     env = cli_env(
         ABXPKG_LIB_DIR=str(shared_lib),
         CHROME_HEADLESS="True",
@@ -395,14 +398,6 @@ def browser_runtime(initialized_archive: Path):
         CHROME_ISOLATION="snapshot",
     )
     env.pop("CHROME_BINARY", None)
-    install_result = run_archivebox_cmd(
-        ["install", "chrome"],
-        cwd=initialized_archive,
-        env=env,
-        timeout=900,
-    )
-    assert install_result.returncode == 0, install_result.stderr or install_result.stdout
-
     resolved_env = resolve_abxpkg_chrome_env(shared_lib, env)
 
     return {

archivebox/tests/test_snapshot_service.py

@@ -198,3 +198,62 @@ def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, re
     assert wget_files
     assert any("Root" in path.read_text(encoding="utf-8", errors="ignore") for path in wget_files if path.suffix in (".html", ".txt"))
     assert any(result["plugin"] == "wget" and result["status"] == ArchiveResult.StatusChoices.SUCCEEDED for result in state["results"])
+
+
+@pytest.mark.timeout(180)
+@pytest.mark.parametrize("depth", [0, 1])
+def test_api_queue_hands_storage_creation_to_runner(tmp_path, recursive_test_site, depth):
+    from .conftest import (
+        create_admin_and_token,
+        live_api_request,
+        stop_archivebox_process,
+        start_archivebox_server,
+        wait_for_snapshot_capture,
+    )
+
+    init_archive(tmp_path)
+    port = get_free_port()
+    env = cli_env(port=port, server=True, PLUGINS="wget,parse_html_urls")
+    token = create_admin_and_token(tmp_path)
+    url = recursive_test_site["root_url"]
+    server = start_archivebox_server(tmp_path, env=env, port=port, log_name="api-storage-handoff.log")
+    try:
+        response = live_api_request(
+            port,
+            "post",
+            "/api/v1/cli/add",
+            api_token=token,
+            json={"urls": [url], "plugins": "wget,parse_html_urls", "tag": "handoff", "depth": depth},
+        )
+        assert response.status_code == 200, response.text
+        crawl_id = response.json()["result"]["crawl_id"]
+        metadata = live_api_request(
+            port,
+            "post",
+            "/api/v1/core/snapshots",
+            api_token=token,
+            json={"crawl_id": crawl_id, "url": url, "title": "Browser title"},
+        )
+        assert metadata.status_code == 200, metadata.text
+        assert "Root" in wait_for_snapshot_capture(tmp_path, url, timeout=120)
+        if depth:
+            assert "About" in wait_for_snapshot_capture(tmp_path, url + "about", timeout=120)
+            child_state = _snapshot_state(tmp_path, url + "about")
+            assert child_state["status"] == Snapshot.StatusChoices.SEALED
+            assert child_state["crawl_link"].is_symlink()
+            assert child_state["crawl_link"].resolve() == child_state["snapshot_dir"].resolve()
+            assert (child_state["snapshot_dir"] / "index.jsonl").is_file()
+        with use_archivebox_db(tmp_path):
+            snapshot = Snapshot.objects.get(pk=metadata.json()["id"])
+            assert snapshot.tags.filter(name="handoff").exists()
+    finally:
+        stop_archivebox_process(server)
+    state = _snapshot_state(tmp_path, url)
+    assert state["status"] == Snapshot.StatusChoices.SEALED
+    assert state["crawl_dir"].is_dir()
+    assert state["snapshot_dir"].is_dir()
+    assert state["crawl_link"].is_symlink()
+    assert state["crawl_link"].resolve() == state["snapshot_dir"].resolve()
+    assert (state["snapshot_dir"] / "index.jsonl").is_file()
+    assert any(row["plugin"] == "wget" and row["status"] == "succeeded" for row in state["results"])
+    assert any("Root" in path.read_text(errors="ignore") for path in (state["snapshot_dir"] / "wget").rglob("*.html"))

archivebox/tests/test_takeover_util_2.py

@@ -175,6 +175,10 @@ def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive
     worker_text = worker_config.read_text(encoding="utf-8")
     command = next(line.partition("=")[2] for line in worker_text.splitlines() if line.startswith("command="))
     assert shlex.split(command)[0] == str(sonic_binary), worker_text
+    supervisor_pids = re.findall(r"Supervisord connected \(pid=(\d+)\)", result.stderr)
+    assert supervisor_pids, result.stderr
+    assert all(not pid_is_alive(int(pid)) for pid in supervisor_pids), "Search CLI returned while its owned supervisord was still alive"
+    assert result.stdout == "", result.stdout
     from archivebox.workers.supervisord_util import _sonic_worker_bind_target
 
     assert _sonic_worker_bind_target({"name": "worker_sonic", "command": command}) == ("127.0.0.1", sonic_port)
@@ -198,6 +202,7 @@ def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive
         timeout=60,
     )
     assert relative_result.returncode == 0, relative_result.stderr or relative_result.stdout
+    assert relative_result.stdout == "", relative_result.stdout
     relative_worker_text = worker_config.read_text(encoding="utf-8")
     relative_command = next(line.partition("=")[2] for line in relative_worker_text.splitlines() if line.startswith("command="))
     assert Path(shlex.split(relative_command)[0]).name == relative_binary.name, relative_worker_text

archivebox/tests/test_ui_add_view.py

@@ -37,6 +37,12 @@ ADD_FORM_DEFAULTS = {
 }
 
 
+@pytest.fixture(autouse=True)
+def initialized_persona(db):
+    # Match init/server startup; rendering /add/ must no longer create DB rows.
+    return Persona.get_or_create_default()
+
+
 @pytest.fixture
 def admin_user(db):
     return User.objects.create_superuser(

archivebox/tests/test_ui_add_view_runtime.py

@@ -190,7 +190,7 @@ def assert_no_file_or_shell_payload_snapshots(cwd: Path, *, canary: Path) -> Non
 
 
 @pytest.mark.timeout(180)
-def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_site):
+def test_add_view_queues_work_without_restarting_stopped_runner(tmp_path, recursive_test_site):
     init_archive(tmp_path)
 
     port = get_free_port()
@@ -239,15 +239,30 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s
         )
         assert response.status_code in (302, 303), response.text
 
-        assert _worker_state(tmp_path, "worker_runner") == "RUNNING"
+        assert _worker_state(tmp_path, "worker_runner") != "RUNNING"
         with use_archivebox_db(tmp_path):
             crawl = Crawl.objects.order_by("-created_at").first()
             assert crawl is not None
             assert crawl.tags_str == "restart-supervised-runner"
             assert crawl.urls == recursive_test_site["root_url"]
+            assert crawl.status == Crawl.StatusChoices.QUEUED
+            assert not crawl.output_dir.exists()
     finally:
         stop_server(tmp_path)
 
+    from .conftest import stop_archivebox_process, wait_for_snapshot_capture
+
+    server = start_archivebox_server(tmp_path, env=env, port=port, log_name="resumed-add-server.log")
+    try:
+        assert "Root" in wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=120)
+    finally:
+        stop_archivebox_process(server)
+    with use_archivebox_db(tmp_path):
+        snapshot = Snapshot.objects.get(crawl_id=crawl.id, url=recursive_test_site["root_url"])
+        assert snapshot.status == Snapshot.StatusChoices.SEALED
+        assert snapshot.archiveresult_set.filter(plugin="wget", status=ArchiveResult.StatusChoices.SUCCEEDED).exists()
+        assert any("Root" in path.read_text(errors="ignore") for path in (snapshot.output_dir / "wget").rglob("*.html"))
+
 
 def _login_to_add_view(port: int) -> tuple[requests.Session, requests.Response, str]:
     # Browsers resolve *.localhost themselves; system DNS in stock containers

archivebox/tests/test_ui_admin_crawl.py

@@ -343,6 +343,7 @@ def test_discovered_snapshots_inherit_current_crawl_tags(crawl):
     assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl", "discovered"}
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, admin_user):
     crawl = Crawl.objects.create(
         urls="https://example.com/remove-me",
@@ -352,6 +353,9 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
         crawl=crawl,
         url="https://example.com/remove-me",
     )
+    output_dir = snapshot.output_dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "captured.txt").write_text("remove this capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -362,11 +366,13 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
     assert response.status_code == 200
     assert response.json()["ok"] is True
     assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
+    assert not output_dir.exists()
 
     crawl.refresh_from_db()
     assert "https://example.com/remove-me" not in crawl.urls
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(client, admin_user):
     crawl = Crawl.objects.create(
         urls=("https://cdn.example.com/asset.js\nhttps://cdn.example.com/second.js\nhttps://example.com/root"),
@@ -382,6 +388,12 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
         url="https://example.com/root",
         status=Snapshot.StatusChoices.SEALED,
     )
+    removed_dir = queued_snapshot.output_dir
+    removed_dir.mkdir(parents=True, exist_ok=True)
+    (removed_dir / "captured.txt").write_text("excluded capture")
+    preserved_file = preserved_snapshot.output_dir / "captured.txt"
+    preserved_file.parent.mkdir(parents=True, exist_ok=True)
+    preserved_file.write_text("preserved capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -401,6 +413,8 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
     assert "https://example.com/root" in crawl.urls
     assert not Snapshot.objects.filter(pk=queued_snapshot.pk).exists()
     assert Snapshot.objects.filter(pk=preserved_snapshot.pk).exists()
+    assert not removed_dir.exists()
+    assert preserved_file.read_text() == "preserved capture"
 
 
 def test_snapshot_from_json_trims_markdown_suffixes_on_discovered_urls(crawl):

archivebox/tests/test_ui_admin_links.py

@@ -287,6 +287,7 @@ def test_snapshot_file_icons_link_to_migrated_root_outputs_and_show_all_plugins(
 
     snapshot = _create_snapshot()
     output_path = Path(snapshot.output_dir) / "singlefile.html"
+    output_path.parent.mkdir(parents=True, exist_ok=True)
     output_path.write_text("<html>migrated singlefile</html>", encoding="utf-8")
     root_result = ArchiveResult.objects.create(
         snapshot=snapshot,

archivebox/tests/test_ui_admin_snapshot.py

@@ -26,6 +26,65 @@ pytestmark = pytest.mark.django_db(transaction=True)
 REPO_ROOT = Path(__file__).resolve().parents[2]
 
 
+def test_bulk_snapshot_delete_keeps_rows_until_worker_verifies_files(client, admin_user, crawl):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.runner import run_pending_crawls
+
+    snapshots = [Snapshot.objects.create(url=f"https://example.com/delete/{i}", crawl=crawl) for i in range(2)]
+    output_dirs = [snapshot.output_dir for snapshot in snapshots]
+    for snapshot, output_dir in zip(snapshots, output_dirs):
+        (output_dir / "wget").mkdir(parents=True, exist_ok=True)
+        (output_dir / "wget" / "index.html").write_text("real saved output")
+        ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    ids = [snapshot.pk for snapshot in snapshots]
+    client.force_login(admin_user)
+    response = client.post(
+        reverse("admin:core_snapshot_changelist"),
+        {"action": "delete_snapshots", ACTION_CHECKBOX_NAME: [str(pk) for pk in ids]},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302
+    assert Snapshot.objects.filter(pk__in=ids, status="deleting").count() == 2
+    assert ArchiveResult.objects.filter(snapshot_id__in=ids).count() == 2
+    assert all((path / "wget" / "index.html").is_file() for path in output_dirs)
+    pending = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST)
+    assert pending.status_code == 200
+    assert "Deleting" in pending.content.decode()
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk__in=ids).exists()
+    assert not ArchiveResult.objects.filter(snapshot_id__in=ids).exists()
+    assert not any(path.exists() for path in output_dirs)
+
+
+def test_snapshot_delete_preserves_rows_when_real_filesystem_cleanup_fails(snapshot):
+    from archivebox.core.models import ArchiveResult, Snapshot
+
+    output_dir = snapshot.output_dir
+    protected = output_dir / "wget"
+    protected.mkdir(parents=True, exist_ok=True)
+    saved_file = protected / "index.html"
+    saved_file.write_text("must not lose the row before this file is removed")
+    result = ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    snapshot_id = snapshot.pk
+    protected.chmod(0o500)
+    try:
+        with pytest.raises(PermissionError):
+            Snapshot.objects.filter(pk=snapshot_id).delete()
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert ArchiveResult.objects.filter(pk=result.pk).exists()
+        assert saved_file.is_file()
+        from archivebox.services.runner import run_pending_crawls
+
+        run_pending_crawls(maintenance_only=True)
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert saved_file.is_file()
+    finally:
+        protected.chmod(0o700)
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+    assert not output_dir.exists()
+
+
 def test_failed_hook_partial_file_does_not_become_successful_card(snapshot, cached_abxpkg_lib_dir):
     partial = Path(snapshot.output_dir) / "title" / "partial.html"
     partial.parent.mkdir(parents=True, exist_ok=True)

archivebox/tests/test_ui_config_views.py

@@ -7,7 +7,7 @@ from archivebox.plugins import views as plugin_views
 from archivebox.core import views as core_views
 from archivebox.config import CONSTANTS
 from archivebox.machine.models import Machine
-from archivebox.tests.conftest import install_real_binary
+from archivebox.tests.conftest import ADMIN_TEST_HOST, install_real_binary
 
 
 pytestmark = pytest.mark.django_db
@@ -75,18 +75,42 @@ def test_binaries_list_view_uses_db_version_and_hides_youtube_dl_alias(admin_req
 
     assert len(context["table"]["Binary Name"]) == 1
     assert str(context["table"]["Binary Name"][0].link_item) == "yt-dlp"
-    assert context["table"]["Found Version"][0] == f"✅ {binary.version}"
+    assert context["table"]["Recorded Version"][0] == binary.version
     assert context["table"]["Provided By"][0] == binary.binprovider
-    assert context["table"]["Found Abspath"][0] == binary.abspath
+    assert context["table"]["Recorded Abspath"][0] == binary.abspath
 
 
 def test_binaries_list_view_only_shows_persisted_records(admin_request):
     context = config_views.binaries_list_view.__wrapped__(admin_request("/admin/environment/binaries/"))
 
     assert context["table"]["Binary Name"] == []
-    assert context["table"]["Found Version"] == []
+    assert context["table"]["Recorded Version"] == []
     assert context["table"]["Provided By"] == []
-    assert context["table"]["Found Abspath"] == []
+    assert context["table"]["Recorded Abspath"] == []
+
+
+@pytest.mark.django_db(transaction=True)
+def test_binaries_page_labels_removed_installation_as_recorded_metadata(admin_client, machine, hermetic_lib_dir):
+    from pathlib import Path
+
+    binary = install_real_binary("node", machine=machine)
+    binary_path = Path(binary.abspath)
+    assert binary_path.is_relative_to(hermetic_lib_dir)
+    assert binary_path.is_symlink()
+    binary_path.unlink()
+    assert not binary_path.exists()
+
+    response = admin_client.get("/admin/environment/binaries/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    html = response.content.decode()
+    assert "/admin/environment/binaries/node/" in html
+    assert "Recorded Version" in html
+    assert "Recorded Abspath" in html
+    assert binary.version in html
+    assert str(binary_path) in html
+    assert f"✅ {binary.version}" not in html
+    binary.refresh_from_db()
+    assert binary.abspath == str(binary_path)
 
 
 @pytest.mark.django_db(transaction=True)

bin/lychee_preprocess.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env -S uv run --no-project --script
+import pathlib
+import re
+import sys
+from urllib.parse import unquote, urlsplit
+
+p = pathlib.Path(sys.argv[1])
+text = p.read_text()
+if p.suffix not in (".md", ".html", ".rst"):
+    # Preserve source line numbers, but feed only literal documentation URLs to Lychee.
+    pattern = r'https?://(?:github\.com/(?:ArchiveBox|pirate)/ArchiveBox(?:/wiki|/blob/|/?#)|raw\.githubusercontent\.com/wiki/ArchiveBox/ArchiveBox/|docs\.archivebox\.io)[^\s<>\)\]"\x27`\\\[]*'
+    text = "\n".join(" ".join(u for u in re.findall(pattern, line) if not re.search(r"[{}]", u)) for line in text.splitlines())
+if p.suffix == ".md" and p.parent.name == "docs":
+    pages = {page.stem.lower(): page.stem for page in p.parent.glob("*.md")}
+
+    def wikilink(match):
+        label, _, target = match[1].partition("|")
+        page, sep, fragment = (target or label).replace(" ", "-").partition("#")
+        page = pages.get(page.lower(), page)
+        return f"[{label}]({page}.md{sep}{fragment})"
+
+    text = re.sub(r"\[\[([^]\n]+)\]\]", wikilink, text)
+if p.suffix == ".md":
+    # Lychee folds local Markdown fragments to lowercase; GitHub heading IDs are lowercase.
+    # Preserve explicitly declared mixed-case HTML IDs, but reject heading case typos.
+    errors = []
+    for match in re.finditer(r'\]\(([^\s)]+)\)|href=["\']([^"\']+)', text):
+        url = urlsplit(match[1] or match[2])
+        fragment = unquote(url.fragment)
+        if url.scheme or url.netloc or fragment == fragment.lower():
+            continue
+        target = p.parent / unquote(url.path) if url.path else p
+        if not target.suffix:
+            target = target.with_suffix(".md")
+        if target.suffix != ".md" or not target.is_file():
+            continue  # Lychee handles missing files and non-Markdown targets.
+        ids = re.findall(r'(?:id|name)=["\']([^"\']+)', target.read_text())
+        if fragment not in ids:
+            line = text[: match.start()].count("\n") + 1
+            errors.append(f"{p}:{line}: heading fragment #{fragment} must match lowercase GitHub IDs")
+    if errors:
+        sys.exit("\n".join(errors))
+print(text)

docs/Configuration.md

@@ -138,7 +138,7 @@ archivebox add --persona=personal https://members.example.com/feed
 **Possible Values:** [`Default`]/`personal`/`work`/...
 The persona profile used when no explicit persona is selected for a new crawl. The selected persona is stored on the Crawl row; `DEFAULT_PERSONA` is not duplicated into `Crawl.config`. Personas bundle a Chrome user-data-dir, a `cookies.txt`, auth state, a user-agent, and any other per-identity config into a single named profile, letting you swap between archiving contexts (logged-out vs. signed-into-work-account vs. signed-into-personal-account) without manually juggling files.
 
-ArchiveBox auto-creates the named persona on disk if it doesn't already exist. See the [Personas wiki page](https://github.com/ArchiveBox/ArchiveBox/wiki/Personas) for the full directory layout.
+ArchiveBox auto-creates the named persona on disk if it doesn't already exist. See the [Chromium profile setup guide](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile) for profile setup instructions.
 
 *Related options:*
 [`COOKIES_FILE`](#cookies_file)
@@ -588,7 +588,7 @@ When `True`, every LDAP user who successfully authenticates is auto-promoted to
 *Options for the on-disk layout, file permissions, and temp/lib directories that ArchiveBox reads and writes during archiving.*
 
 > [!NOTE]
-> Docker-only [`PUID` and `PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid) are entrypoint environment variables, not ArchiveBox configuration keys.
+> Docker-only [`PUID` and `PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues) are entrypoint environment variables, not ArchiveBox configuration keys.
 
 ---
 <a id="dir_output_permissions"></a>
@@ -664,7 +664,7 @@ ArchiveBox stores all of its index metadata in a single SQLite database file (`i
 > These are advanced operator tuning options. If you are not actively diagnosing `database is locked` errors or planning a non-default storage layout, you can safely leave everything in this section at its default.
 
 *Learn more:*
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#sqlite-database-is-locked
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database-and-filesystem-contention-issues-when-running-multiple-archivebox-processes
 - https://www.sqlite.org/wal.html
 - https://www.sqlite.org/pragma.html
 

docs/Docker.md

@@ -8,16 +8,16 @@ Running ArchiveBox with Docker allows you to manage it in a container without ex
 
 <img src="https://imgur.zervice.io/qFAPRwC.png" width="20%" align="right"/>
 
-- [Overview](#Overview)
+- [Overview](#overview)
 - [Docker Compose](#docker-compose) ⭐️ (recommended)
   - [Setup](#setup)
-  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-docker-compose-%EF%B8%8F)
+  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-docker-compose-%EF%B8%8F)
   - [Usage](#usage)
   - [Accessing the data](#accessing-the-data)
   - [Configuration](#configuration)
 - [Plain Docker](#docker)
   - [Setup](#setup-1)
-  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-plain-docker)
+  - [Upgrading](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-plain-docker)
   - [Usage](#usage-1)
   - [Accessing the data](#accessing-the-data-1)
   - [Configuration](#configuration-1)
@@ -82,7 +82,7 @@ docker compose exec archivebox archivebox config --set SEARCH_BACKEND_ENGINE=rip
 
 ### Upgrading
 
-See the wiki page on [Upgrading or Merging Archives: Upgrading with Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-docker-compose-%EF%B8%8F) for instructions. ➡️
+See the wiki page on [Upgrading or Merging Archives: Upgrading with Docker Compose](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-docker-compose-%EF%B8%8F) for instructions. ➡️
 
 <br/>
 
@@ -220,13 +220,13 @@ Then open `/admin/` on the hostname or IP used to reach ArchiveBox (local exampl
 
 *(You can create a collection in any directory you want, `~/archivebox/data` is just used as an example here)*
 
-If you encounter permissions issues, make sure the mounted data directory is writable by its intended owner. Docker startup uses explicit [`PUID`/`PGID`](#puid--pgid) values when supplied, otherwise the first non-root owner detected from the existing collection or the default `archivebox` user when the data directory is root-owned.
+If you encounter permissions issues, make sure the mounted data directory is writable by its intended owner. Docker startup uses explicit [`PUID`/`PGID`](#docker-puid--pgid) values when supplied, otherwise the first non-root owner detected from the existing collection or the default `archivebox` user when the data directory is root-owned.
 
 <br/>
 
 ### Upgrading
 
-See the wiki page on [Upgrading or Merging Archives: Upgrading with plain Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#upgrading-with-plain-docker) for instructions. ➡️
+See the wiki page on [Upgrading or Merging Archives: Upgrading with plain Docker](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading#upgrading-with-plain-docker) for instructions. ➡️
 
 <br/>
 

docs/Install.md

@@ -11,8 +11,8 @@ ArchiveBox is primarily distributed as a Python package installed with `uv`, but
    - **[Option A. Docker / Docker Compose ⭐️](#option-a-docker--docker-compose-setup-%EF%B8%8F)**
    - [Option B. uv Install Shortcut](#option-b-uv-install-shortcut)
    - [Option C. System Package Manager Setup](#option-c-bare-metal-setup)
-     - *[Upgrading ArchiveBox to a new version](#upgrading-archivebox-to-a-new-version)*
- - *[Next Steps](#next-steps)*
+     - *[Upgrading ArchiveBox to a new version](#next-steps-upgrading-archivebox-to-a-new-version)*
+ - *[Next Steps](#next-steps-add-some-urls-to-archive-and-try-out-cli--web-ui)*
 
 
 ## Supported Systems
@@ -99,7 +99,7 @@ See our [Dependencies](https://github.com/ArchiveBox/ArchiveBox#dependencies) do
 <img src="https://avatars0.githubusercontent.com/u/1503512?s=200&v=4" width="100px" align="right"/>
 
 **More info:**
- - For help installing these, see the [Manual Setup](#manual-setup), [[Troubleshooting]] and [[Chromium Install]] pages.
+ - For help installing these, see the [Manual Setup](#option-c-bare-metal-setup), [[Troubleshooting]] and [[Chromium Install]] pages.
  - For per-plugin binary and enable/disable options (CHROME_BINARY, RIPGREP_BINARY, `<plugin>_ENABLED`, etc.) see the [abx-plugins config reference](https://plugins.archivebox.io/).
 
 

docs/Release-Notes-v0.9.md

@@ -148,7 +148,7 @@ This is intentionally limited to advisory threads where the maintainer explicitl
 
 - [ArchiveBox documentation](https://docs.archivebox.io/)
 - [ArchiveBox browser extension](https://github.com/ArchiveBox/archivebox-browser-extension)
-- [ArchiveBox REST API documentation](https://docs.archivebox.io/dev/#rest-api)
+- [ArchiveBox REST API documentation](https://docs.archivebox.io/dev/apidocs/archivebox/archivebox.api.html)
 - [ArchiveBox releases](https://github.com/ArchiveBox/ArchiveBox/releases)
 - [Full source comparison from v0.7.4 to the current v0.9 release candidate](https://github.com/ArchiveBox/ArchiveBox/compare/v0.7.4...v0.9.35rc410)
 

docs/Security-Overview.md

@@ -46,7 +46,7 @@ archivebox persona create --import=chrome personal
 archivebox add --persona=personal 'https://members.example.com/'
 ```
 
-To get started, import a dedicated browser profile into a [persona](https://github.com/ArchiveBox/ArchiveBox/wiki/Personas). A persona keeps its Chrome profile and `cookies.txt` together and applies the same identity consistently across extractors.
+To get started, import a dedicated browser profile into a [persona](https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile). A persona keeps its Chrome profile and `cookies.txt` together and applies the same identity consistently across extractors.
 
 ➡️ For full instructions on setting up a Chromium user profile see here: https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile
 
@@ -134,9 +134,9 @@ The ArchiveBox database is an unencrypted, uncompressed SQLite3 `index.sqlite3`
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#modify-the-archivebox-sqlite3-db-directly
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#example-adding-a-new-user-with-a-hashed-password
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication#username--password-the-default
 
 ### Filesystem
 
@@ -150,11 +150,11 @@ How much are you planning to archive? Only a few bookmarked articles, or thousan
 
 Consider what permissioning to apply to your archive folder carefully. Limit access to the fewest possible users by checking folder ownership and setting [`OUTPUT_PERMISSIONS`](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#output_permissions) accordingly. Generally the `index.sqlite3` file, `archive/` folder, and `ArchiveBox.conf` file must all be owned and writable by the `archivebox` user or a dedicated non-root user.
 
-When running with Docker, the entrypoint uses explicit [`PUID`/`PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#puid--pgid) values when supplied, otherwise the existing non-root owner of the mounted data directory or the image's `archivebox` user.
+When running with Docker, the entrypoint uses explicit [`PUID`/`PGID`](https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#docker-permissions-issues) values when supplied, otherwise the existing non-root owner of the mounted data directory or the image's `archivebox` user.
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout
 - https://github.com/ArchiveBox/ArchiveBox#output-formats
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#filesystem-doesnt-support-fsync-eg-network-mounts
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#filesystem-doesnt-support-fsync-eg-network-mounts
 - https://github.com/ArchiveBox/ArchiveBox#storage-requirements

docs/Setting-Up-Storage.md

@@ -18,7 +18,7 @@ services:
 <h4>Related Docs</h4>
 <ul>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox#archive-layout">README: Archive Layout</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Wiki: Usage (Disk Layout)</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Wiki: Usage (Disk Layout)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#large-archives">Wiki: Usage (Large Archives)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder">Wiki: Security Overview (Output Folder)</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Wiki: Publishing Your Archive</a></li>

docs/Setting-up-Authentication.md

@@ -91,7 +91,7 @@ LDAP_LASTNAME_ATTR="sn"
 LDAP_EMAIL_ATTR="mail"
 ```
 
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#ldap
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#ldap-settings
 - https://github.com/ArchiveBox/ArchiveBox/pull/1214
 - https://github.com/django-auth-ldap/django-auth-ldap#example-configuration
 - https://jumpcloud.com/blog/what-is-ldap-authentication

docs/Troubleshooting.md

@@ -4,11 +4,11 @@
 
 What are you having an issue with?:
 
-- [Installing ArchiveBox](#Installing)
+- [Installing ArchiveBox](#installing)
 - [Upgrading ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives)
 - [Configuring ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration)
-- [Archiving content with ArchiveBox](#Archiving)
-- [Hosting your collection publicly](#Hosting-the-Archive)
+- [Archiving content with ArchiveBox](#archiving)
+- [Hosting your collection publicly](#hosting-the-archive)
 - [Database and filesystem issues](#database)
 
 ---
@@ -296,5 +296,5 @@ See here for more info:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Merging-Collections
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#python-shell-usage
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#run-archivebox-as-an-unprivileged-user
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder

docs/Upgrading.md

@@ -61,7 +61,7 @@ As of v0.4 and above, ArchiveBox uses the Django migrations system for determini
 
 More info:
 - https://docs.djangoproject.com/en/4.0/topics/migrations/
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-migrations-errors-or-upgrade-issues
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database-migrations-errors-or-upgrade-issues
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting
 
 ---
@@ -146,7 +146,7 @@ archivebox status      # check that everything succeeded
 
 More info:
 - https://github.com/ArchiveBox/ArchiveBox#-package-manager-setup
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#manual-setup
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Install#option-c-bare-metal-setup
 - https://github.com/ArchiveBox/homebrew-archivebox
 - https://github.com/ArchiveBox/debian-archivebox
 - https://aur.archlinux.org/packages/archivebox

docs/Usage.md

@@ -6,10 +6,10 @@
 
 <img src="https://imgur.zervice.io/aQZZcku.png" width="20%" align="right"/>
 
-- [CLI Usage](#CLI-Usage): Docs and examples for the ArchiveBox command line interface.
-- [Admin UI Usage](#UI-Usage): Docs and screenshots for the outputted HTML archive interface.
-- [Browser Extension Usage](#Browser-Extension-Usage): Docs and screenshots for the outputted HTML archive interface.
-- [Disk Layout](#Disk-Layout): Description of the archive folder structure and contents.
+- [CLI Usage](#cli-usage): Docs and examples for the ArchiveBox command line interface.
+- [Admin UI Usage](#ui-usage): Docs and screenshots for the outputted HTML archive interface.
+- [Browser Extension Usage](#browser-extension-usage): Docs and screenshots for the outputted HTML archive interface.
+- [Disk Layout](#disk-layout): Description of the archive folder structure and contents.
 
 **Related:**
 
@@ -36,10 +36,10 @@ You can share a single archivebox data directory between Docker and non-Docker i
 
 For more examples see [README: Usage](https://github.com/ArchiveBox/ArchiveBox#%EF%B8%8F-cli-usage) and [[Docker]] pages.
 
-- [Run ArchiveBox with configuration options](#Run-ArchiveBox-with-configuration-options)
-- [Import a single URL](#Import-a-single-URL)
-- [Import a list of URLs from a text file](#Import-a-list-of-URLs-from-a-text-file)
-- [Import list of links from browser history](#Import-list-of-links-from-browser-history)
+- [Run ArchiveBox with configuration options](#run-archivebox-with-configuration-options)
+- [Import a single URL](#import-a-single-url)
+- [Import a list of URLs from a text file](#import-a-list-of-urls-from-a-text-file)
+- [Import list of links from browser history](#import-list-of-links-from-browser-history)
 
 ---
 
@@ -146,7 +146,7 @@ archivebox server 0.0.0.0:5797     # start the archivebox web server
 
 Open <http://admin.archivebox.localhost:5797/admin/> in a browser to create the first admin and finish web setup. To create additional accounts from the CLI, use `archivebox manage createsuperuser`.
 
-*See the [Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions) and [Security Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#archiving-private-content) for more info...*
+*See the [Configuration Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#permissions) and [Security Wiki](https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#archiving-content-behind-log-ins--advanced-users-only) for more info...*
 
 Or if you prefer to generate a [static HTML index](https://github.com/ArchiveBox/ArchiveBox#static-archive-exporting) instead of using the built-in web server, you can run `archivebox list --html --with-headers > ./index.html` and then open `./index.html` in a browser.  You should see something [like this](https://demo.archivebox.io).
 
@@ -254,8 +254,8 @@ Simply back up the entire `data/` folder to back up your archive, e.g. `zip -r d
 For more info about ArchiveBox's database/filesystem layout and troubleshooting steps:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#modify-the-archivebox-sqlite3-db-directly
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
 
 
 ### Large Archives
@@ -280,7 +280,7 @@ which are already in the index.
 
 For more info about troubleshooting filesystem permissions, performance, or issues when running on a NAS:
 - https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#output-folder
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
 
 <br/>
 
@@ -302,9 +302,9 @@ UPDATE auth_user SET email = 'someNewEmail@example.com' WHERE username = 'someUs
 ```
 
 More info:
-- https://github.com/ArchiveBox/ArchiveBox#-sqlpythonfilesystem-usage
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#modify-the-archivebox-sqlite3-db-directly
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#database-troubleshooting
+- https://github.com/ArchiveBox/ArchiveBox#usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#sql-shell-usage
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#database
 - https://stackoverflow.com/questions/1074212/how-can-i-see-the-raw-sql-queries-django-is-running
 - https://adamobeng.com/wddbfs-mount-a-sqlite-database-as-a-filesystem/
 
@@ -341,7 +341,7 @@ $ archivebox shell
 ```
 
 For more info and example usage:
-- https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives#example-adding-a-new-user-with-a-hashed-password
+- https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-up-Authentication#username--password-the-default
 - https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/cli/
 - https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/config/common.py
 - https://github.com/ArchiveBox/ArchiveBox/blob/dev/archivebox/core/models.py

docs/Web-Archiving-Community.md

@@ -16,10 +16,10 @@ Whether you want to learn which organizations are the big players in the web arc
 
 <img src="https://imgur.zervice.io/duS8Lm7.png" width="200px" align="right" style="float: right; margin: 5px"/>
 
-- [The Master Lists](#The-Master-Lists)  
+- [The Master Lists](#the-master-lists)  
   *Community-maintained indexes of web archiving tools and groups by IIPC, COPTR, ArchiveTeam, Wikipedia, & the ASA.* 
 
-- [Web Archiving Software](#Web-Archiving-Projects)  
+- [Web Archiving Software](#web-archiving-projects)  
   *Open source tools and projects in the internet archiving space.*
   - [Bookmarking Services](#bookmarking-services)
   - [Well-Known Open Source Projects](#from-the-archiveorg--archive-it-teams)
@@ -27,14 +27,14 @@ Whether you want to learn which organizations are the big players in the web arc
   - [ArchiveBox Alternatives](#other-archivebox-alternatives)
   - [Smaller Utilities](#smaller-utilities)
 
-- [Reading List](#Reading-List)  
+- [Reading List](#reading-list)  
   *Articles, posts, and blogs relevant to ArchiveBox and web archiving in general.*
-  - [Blogs](#Blogs)
-  - [Articles](#Articles)
+  - [Blogs](#blogs-friends-of-archivebox)
+  - [Articles](#articles-we-like-about-internet-archiving)
   - [ArchiveBox-Specific Posts, Tutorials, and Guides](#archivebox-specific-posts-tutorials-and-guides)
   - [ArchiveBox Discussions in News & Social Media](#archivebox-discussions-in-news--social-media)
 
-- [Communities](#Communities)  
+- [Communities](#communities)  
   *A collection of the most active internet archiving communities and initiatives.*
   - [Most Active Web-Archiving Communities](#most-active-communities)
   - [Other Web Archiving Communities](#web-archiving-communities)
@@ -504,5 +504,5 @@ You can find more organizations and initiatives on these other lists:
 [![](https://img.shields.io/badge/Donate-Archive.org-%23115D76.svg)](https://archive.org/donate/)
 
 <br/><br/>
-<small><a href="#contents">^ &nbsp; Back to Top &nbsp; ^</a></small>
+<small><a href="#web-archiving-community">^ &nbsp; Back to Top &nbsp; ^</a></small>
 </div>

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.71",
+  "version": "0.9.72rc45",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

publicsite/ar/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">تكوين ويكي</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">إعدادات Docker</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">تثبيت التبعيات</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">تثبيت التبعيات</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">خيارات تثبيت Chromium</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">ملفات تعريف الارتباط وملامح المتصفح المسجلة الدخول</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">تثبيت استكشاف الأخطاء وإصلاحها</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">قائمة archivebox -- json -- مع رؤوس&amp;gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">قائمة archivebox -- csv = الطابع الزمني ، عنوان url ، العنوان&amp;gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">مستندات تخطيط القرص</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">مستندات تخطيط القرص</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">خلفية تخزين: SMB ، NFS ، S3 ، B2 ، محرك جوجل</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">نشر أرشيفك</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">ترقية أو دمج المحفوظات</a></li>

publicsite/es/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Wiki de configuración</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Configuración de la ventana acoplable</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Instalar dependencias</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Instalar dependencias</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Opciones de instalación de Chrome</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookies y perfiles de navegador registrados</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Solución de problemas de instalación</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">lista de cuadros de archivo --json --con-encabezados &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">lista de cuadros de archivo --csv=marca de tiempo,url,título &gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Documentos de diseño de disco</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Documentos de diseño de disco</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Backends de almacenamiento: SMB, NFS, S3, B2, Google Drive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Publicando su archivo</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Actualizar o fusionar archivos</a></li>

publicsite/fr/index.html

@@ -438,7 +438,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <ul class="readme-list">
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Wiki de configuration</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Configuration du menu Docker</a></li>
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Installer les dépendances</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Installer les dépendances</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Options d'installation de Chrome</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookies et profils de navigateur connectés</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Dépannage des installations</a></li>
@@ -546,7 +546,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">liste des boîtes d'archives --json --with-headers &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">liste des boîtes d'archives --csv=horodatage,url,titre &gt; index.csv</span></pre></div></div>
 <ul class="readme-list">
-<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Documentation sur la configuration du disque</a></li>
+<li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Documentation sur la configuration du disque</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Backends de stockage : SMB, NFS, S3, B2, Google Drive</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Publier vos archives</a></li>
 <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Mise à niveau ou fusion d'archives</a></li>

publicsite/index.html

@@ -432,7 +432,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
           <ul class="readme-list">
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration">Configuration wiki</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#configuration">Docker configuration</a></li>
-            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#dependencies">Install dependencies</a></li>
+            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Install#3-install-runtime-dependencies-using-archivebox-install">Install dependencies</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install">Chromium install options</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Chromium-Install#setting-up-a-chromium-user-profile">Cookies and logged-in browser profiles</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Troubleshooting#installing">Troubleshooting installs</a></li>
@@ -542,7 +542,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
 <span class="ab-prompt">$</span> <span class="ab-cmd">archivebox list --json --with-headers &gt; index.json</span>
 <span class="ab-prompt">$</span> <span class="ab-cmd">archivebox list --csv=timestamp,url,title &gt; index.csv</span></pre></div></div>
           <ul class="readme-list">
-            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#Disk-Layout">Disk layout docs</a></li>
+            <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage#disk-layout">Disk layout docs</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Setting-Up-Storage">Storage backends: SMB, NFS, S3, B2, Google Drive</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Publishing-Your-Archive">Publishing your archive</a></li>
             <li><a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Upgrading-or-Merging-Archives">Upgrading or merging archives</a></li>