Bump release version to 0.9.65rc15

.github/STAGING.md

@@ -0,0 +1,31 @@
+# Stable releases require live dev acceptance
+
+The order is **dev CI → dev image publication → automatic Cabbage and DigestBox
+deployment → real capture and browser acceptance → stable publication**. Never
+substitute a published PyPI wheel, healthy container, or local build for this
+acceptance. The release workflow blocks *before* a stable upload unless both
+environments have successful evidence from `staging-acceptance.yml` matching all
+tracked source and dependency pins. Only ArchiveBox's own version fields may
+change between dev and stable.
+
+The staging workflow observes the existing image watchers; it cannot deploy or
+restart the applications. On both hosts, install `bin/staging-acceptance-host.sh`
+as root-owned mode 0755 `/usr/local/sbin/archivebox-staging-acceptance`. Its hash
+is checked against the candidate. A dedicated SSH key must use
+`restrict,command="/usr/local/sbin/archivebox-staging-acceptance"` in authorized
+keys. It accepts only `inspect <source-sha>` and `capture <source-sha>`; it cannot
+run caller-provided commands or archive arbitrary URLs.
+
+The `staging` GitHub environment is restricted to dev/main branches. It contains
+`STAGING_SSH_KEY` and `STAGING_KNOWN_HOSTS`, with independently verified host keys.
+Only trusted release events use the private ugNAS runner route. Never expose
+these credentials to PR jobs or copy an operator's existing root key.
+
+Acceptance creates one tagged background crawl per source through the normal
+add/runner path, enabling every snapshot extractor, including TLSNotary. It
+checks installed pins, completed outputs and their files, unexpected/missing
+plugin results, host OOM kills, and real rendered snapshot pages. Logs, result
+inventories and screenshots are retained as Actions artifacts. Failures remain
+failures; do not publish a success deployment record to bypass them. No code
+mounts are allowed. Both matrix jobs must pass before their records authorize a
+stable upload. Queued capture progress is polled; failed tests are not retried.

.github/actionlint.yaml

@@ -0,0 +1,3 @@
+self-hosted-runner:
+  labels:
+    - ugnas-linux

.github/workflows/docker.yml

@@ -552,6 +552,9 @@ jobs:
 
       - name: Export digest
         if: inputs.push_digests
+        env:
+          ABX_DL_IMAGE_REF: ${{ steps.abx_dl_image.outputs.image }}
+          ARTIFACT_NAME: ${{ matrix.artifact_name }}
         shell: bash
         run: |
           set -Eeuo pipefail
@@ -559,6 +562,7 @@ jobs:
           digest="${{ steps.docker_build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
           printf '%s\n' '${{ inputs.source_ref || github.sha }}' > /tmp/digests/source-ref
+          printf '%s\n' "$ABX_DL_IMAGE_REF" > "/tmp/digests/abx-dl-image-$ARTIFACT_NAME"
 
       - name: Upload digest
         if: inputs.push_digests

.github/workflows/release.yml

@@ -144,10 +144,12 @@ jobs:
     runs-on: ubuntu-24.04
     env:
       CI_RUN_ID: ${{ needs.candidate.outputs.artifact_run_id }}
+      DOCKER_DIGEST_RUN_ID: ${{ needs.candidate.outputs.digest_run_id }}
     environment: pypi
     permissions:
       actions: read
       contents: write
+      deployments: read
       id-token: write
     outputs:
       version: ${{ steps.version.outputs.version }}
@@ -171,6 +173,26 @@ jobs:
           version: "0.11.3"
           enable-cache: false
 
+      # This precedes every stable upload, including workflow_dispatch. A green
+      # build is not evidence that our automatically updated test servers ran it.
+      - name: Download tested Docker base-image metadata
+        if: env.RELEASE_BRANCH == 'main'
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          path: ${{ runner.temp }}/tested-docker-artifacts
+          pattern: digest-*
+          merge-multiple: true
+          github-token: ${{ github.token }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.DOCKER_DIGEST_RUN_ID }}
+
+      - name: Require matching acceptance on Cabbage and DigestBox
+        if: env.RELEASE_BRANCH == 'main'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          STAGING_DOCKER_ARTIFACT_DIR: ${{ runner.temp }}/tested-docker-artifacts
+        run: uv run --no-project python bin/verify-staging-release.py
+
       - name: Resolve release binaries through abxpkg
         env:
           ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib

.github/workflows/staging-acceptance.yml

@@ -0,0 +1,81 @@
+name: Staging acceptance
+
+on:
+  workflow_run:
+    workflows: [Release]
+    types: [completed]
+    branches: [dev]
+
+permissions: {}
+
+concurrency:
+  group: staging-acceptance
+  cancel-in-progress: false
+
+jobs:
+  published:
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'dev'
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+    outputs:
+      ready: ${{ steps.image.outputs.ready }}
+    steps:
+      - id: image
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_RUN_ID: ${{ github.event.workflow_run.id }}
+        run: |
+          set -Eeuo pipefail
+          ready=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID/jobs?per_page=100" --jq 'any(.jobs[]; .name == "Publish tested multiarch image" and .conclusion == "success")')
+          echo "ready=$ready" >> "$GITHUB_OUTPUT"
+  acceptance:
+    # The privileged key is never exposed to PRs or arbitrary dispatch refs.
+    needs: published
+    if: needs.published.outputs.ready == 'true'
+    runs-on: [self-hosted, Linux, X64, ugnas-linux]
+    environment: staging
+    timeout-minutes: 110
+    permissions:
+      contents: read
+      deployments: write
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [cabbage, digestbox]
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e
+        with:
+          version: '0.11.3'
+      - name: Prepare browser and restricted staging identity
+        env:
+          STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
+          STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
+        run: |
+          set -Eeuo pipefail
+          umask 077
+          printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
+          printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
+          uv run --no-project --with playwright==1.63.0 playwright install chromium
+      - name: Verify actual deployment, full captures and rendered snapshots
+        env:
+          GH_TOKEN: ${{ github.token }}
+          STAGING_TARGET: ${{ matrix.target }}
+          STAGING_SHA: ${{ github.event.workflow_run.head_sha }}
+          STAGING_KEY_FILE: ${{ runner.temp }}/staging-key
+          STAGING_KNOWN_HOSTS_FILE: ${{ runner.temp }}/staging-known-hosts
+        run: uv run --no-project --with playwright==1.63.0 python bin/check-staging-deployment.py
+      - name: Remove staging identity
+        if: always()
+        run: rm -f "$RUNNER_TEMP/staging-key" "$RUNNER_TEMP/staging-known-hosts"
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        if: always()
+        with:
+          name: staging-${{ matrix.target }}-${{ github.event.workflow_run.head_sha }}
+          path: staging-evidence/
+          if-no-files-found: error

archivebox/config/django.py

@@ -63,7 +63,14 @@ def setup_django(check_db=False) -> None:
             os.chown(CONSTANTS.DATA_DIR, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
             if CONSTANTS.DATA_DIR.exists():
                 for child in CONSTANTS.DATA_DIR.iterdir():
-                    os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
+                    try:
+                        os.chown(child, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, follow_symlinks=False)
+                    except FileNotFoundError:
+                        # A concurrent SQLite transaction can remove its journal
+                        # between directory enumeration and chown. There is no
+                        # ownership left to fix; other permission errors must
+                        # still fail startup rather than being hidden.
+                        continue
 
     # Suppress the "database access during app initialization" warning
     # This warning can be triggered during django.setup() but is safe to ignore

archivebox/core/migrations/0024_assign_default_crawl.py

@@ -100,11 +100,12 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
     if schema_editor.connection.vendor != "sqlite":
         return
 
-    from django.db import connection
+    from django.utils import timezone
     import uuid as uuid_lib
-    from datetime import datetime
 
+    connection = schema_editor.connection
     cursor = connection.cursor()
+    now = connection.ops.adapt_datetimefield_value(timezone.now())
 
     # Check if there are any snapshots without a crawl
     cursor.execute("SELECT COUNT(*) FROM core_snapshot WHERE crawl_id IS NULL")
@@ -125,14 +126,13 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
             INSERT INTO auth_user (id, password, is_superuser, username, first_name, last_name, email, is_staff, is_active, date_joined)
             VALUES (1, '!', 1, 'system', '', '', '', 1, 1, %s)
         """,
-            [datetime.now().isoformat()],
+            [now],
         )
 
     # Create a default crawl for migrated snapshots.
     # Depending on migration graph order, later crawls migrations may already
     # have removed output_dir by the time this data migration runs.
     crawl_id = uuid_lib.uuid4().hex
-    now = datetime.now().isoformat()
     cursor.execute("PRAGMA table_info(crawls_crawl)")
     crawl_columns = {row[1] for row in cursor.fetchall()}
     default_columns: list[str] = []
@@ -155,9 +155,9 @@ def create_default_crawl_and_assign_snapshots(apps, schema_editor):
             status, retry_at, created_by_id, schedule_id, config, persona_id
         ) VALUES (%s, %s, %s, 0, 0, %s, 0, '', 'Migrated from v0.7.2/v0.8.6',
                   'Auto-created crawl for migrated snapshots', {default_values_sql}
-                  'sealed', %s, 1, NULL, '{{}}', NULL)
+                  'sealed', NULL, 1, NULL, '{{}}', NULL)
     """,
-        [crawl_id, now, now, crawl_urls, now],
+        [crawl_id, now, now, crawl_urls],
     )
 
     # Assign all snapshots without a crawl to the default crawl

archivebox/core/migrations/0055_repair_legacy_retry_timestamps.py

@@ -0,0 +1,60 @@
+"""Repair SQLite queue timestamps written as raw ISO strings by old migrations."""
+
+from django.db import migrations
+from django.utils.dateparse import parse_datetime
+
+
+def repair_legacy_retry_timestamps(apps, schema_editor):
+    connection = schema_editor.connection
+    if connection.vendor != "sqlite":
+        return
+
+    with connection.cursor() as cursor:
+        # 0024 created this Crawl already sealed, but left a due retry marker.
+        # Its orphaned snapshots have their own scheduler rows for maintenance.
+        cursor.execute(
+            """
+            UPDATE crawls_crawl SET retry_at = NULL
+            WHERE status = 'sealed'
+              AND label = 'Migrated from v0.7.2/v0.8.6'
+              AND notes = 'Auto-created crawl for migrated snapshots'
+              AND substr(retry_at, 11, 1) = 'T'
+            """,
+        )
+
+        # SQLite compares stored date strings lexically. A raw `T` timestamp
+        # from a legacy migration sorts differently from Django's space-separated
+        # parameter, so a due row can be selected but never claimed. Preserve
+        # the datetime Django already reads from each row while rewriting its
+        # storage representation through the active database adapter.
+        for table in ("crawls_crawl", "core_snapshot", "machine_binary", "machine_process"):
+            last_id = ""
+            while True:
+                cursor.execute(
+                    f"SELECT id, CAST(retry_at AS TEXT) FROM {table} WHERE id > %s AND substr(retry_at, 11, 1) = 'T' ORDER BY id LIMIT 500",
+                    [last_id],
+                )
+                rows = cursor.fetchall()
+                if not rows:
+                    break
+                for row_id, raw_retry_at in rows:
+                    try:
+                        parsed = parse_datetime(raw_retry_at)
+                    except ValueError:
+                        continue
+                    if parsed is None:
+                        continue
+                    normalized = connection.ops.adapt_datetimefield_value(parsed)
+                    cursor.execute(
+                        f"UPDATE {table} SET retry_at = %s WHERE id = %s AND retry_at = %s",
+                        [normalized, row_id, raw_retry_at],
+                    )
+                last_id = rows[-1][0]
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0054_snapshot_fs_version_index"),
+    ]
+
+    operations = [migrations.RunPython(repair_legacy_retry_timestamps, migrations.RunPython.noop)]

archivebox/progressmonitor/templates/progressmonitor/progress_monitor.html

@@ -1904,12 +1904,22 @@
     const progressEndpoint = monitor.dataset.progressEndpoint || '/progress.json';
 
     function fetchProgress() {
+        // Opening a replay replaces the queued snapshot's progress panel.
+        // Its interval and already-running fetch must stop using that old DOM.
+        if (!monitor.isConnected) {
+            stopPolling();
+            return;
+        }
         fetch(progressEndpoint, { credentials: 'same-origin' })
             .then(response => {
                 if (!response.ok) throw new Error(`HTTP ${response.status}`);
                 return response.json();
             })
             .then(data => {
+                if (!monitor.isConnected) {
+                    stopPolling();
+                    return;
+                }
                 if (data.error) {
                     console.error('Progress API error:', data.error, data.traceback);
                     setOrchestratorState('error', 'Backend not responding');
@@ -1921,6 +1931,10 @@
                 updateProgress(data);
             })
             .catch(error => {
+                if (!monitor.isConnected) {
+                    stopPolling();
+                    return;
+                }
                 console.error('Progress fetch error:', error);
                 setOrchestratorState('error', 'Backend not responding');
                 idleMessage.textContent = 'Fetch Error: ' + error.message;
@@ -1929,7 +1943,7 @@
     }
 
     function startPolling() {
-        if (pollInterval) return;
+        if (pollInterval || !monitor.isConnected) return;
         fetchProgress();
         pollInterval = setInterval(fetchProgress, pollDelayMs);
     }
@@ -2033,7 +2047,13 @@
 
     // Start polling when page loads
     startPolling();
-    setInterval(updateDurationBadges, 1000);
+    const durationInterval = setInterval(() => {
+        if (!monitor.isConnected) {
+            clearInterval(durationInterval);
+            return;
+        }
+        updateDurationBadges();
+    }, 1000);
 
     // Pause polling when tab is hidden
     document.addEventListener('visibilitychange', function() {

archivebox/services/resource_admission.py

@@ -2,9 +2,14 @@
 
 import math
 import os
+import re
+import stat
+import sys
 import time
 from pathlib import Path
 
+import psutil
+
 
 RESOURCE_RECHECK_SECONDS = 2.0
 # Scheduling policy, not a minimum-RAM claim: stop increasing extraction
@@ -12,6 +17,65 @@ RESOURCE_RECHECK_SECONDS = 2.0
 # stalls. Require both recent and rolling pressure so transient reclaim does
 # not indefinitely starve otherwise viable work on small hosts.
 MEMORY_FULL_STALL_PERCENT = 10.0
+ZRAM_DEVICE_NAME = re.compile(r"zram\d+")
+
+
+def zram_swap_identifiers(sys_block: Path = Path("/sys/block")) -> set[str]:
+    """Return zram block names and major:minor IDs known to sysfs."""
+    identifiers = set()
+    for device in sys_block.glob("zram*"):
+        if ZRAM_DEVICE_NAME.fullmatch(device.name):
+            identifiers.add(device.name)
+            try:
+                identifiers.add((device / "dev").read_text().strip())
+            except OSError:
+                pass
+    return identifiers
+
+
+def swap_source_identifiers(filename: str) -> set[str]:
+    """Identify a swap source by its visible name and, for block devices, ID."""
+    identifiers = {Path(filename).name}
+    try:
+        source = os.stat(filename)
+        if stat.S_ISBLK(source.st_mode):
+            identifiers.add(f"{os.major(source.st_rdev)}:{os.minor(source.st_rdev)}")
+    except OSError:
+        pass
+    return identifiers
+
+
+def disk_backed_swap_free_bytes(swaps: str, zram_identifiers: set[str]) -> int:
+    """Sum free active swap except zram, whose capacity is already backed by RAM."""
+    lines = swaps.splitlines()
+    free_kib = 0
+    for line in lines[1:]:  # /proc/swaps starts with a column header
+        fields = line.split()
+        if len(fields) < 5:
+            continue
+        filename, _, size_kib, used_kib, _ = fields[:5]
+        source_ids = swap_source_identifiers(filename)
+        if source_ids & zram_identifiers or any(ZRAM_DEVICE_NAME.fullmatch(name) for name in source_ids):
+            continue
+        try:
+            free_kib += max(0, int(size_kib) - int(used_kib))
+        except ValueError:
+            continue
+    return free_kib * 1024
+
+
+def host_swap_free_bytes(meminfo_swap_free: int) -> int:
+    """Use only independently backed swap as extra capacity on Linux."""
+    if not sys.platform.startswith("linux"):
+        return psutil.swap_memory().free
+    try:
+        swaps = Path("/proc/swaps").read_text()
+    except OSError:
+        # If Linux swap sources cannot be classified, fail closed rather than
+        # treating potentially RAM-backed zram capacity as extra headroom.
+        return 0
+    disk_free = disk_backed_swap_free_bytes(swaps, zram_swap_identifiers())
+    return min(max(0, meminfo_swap_free), disk_free)
 
 
 class ResourceAdmission:
@@ -44,10 +108,73 @@ class ResourceAdmission:
                     cpus = min(cpus, max(1, math.ceil(int(quota) / int(period))))
             except (OSError, ValueError, ZeroDivisionError):
                 continue
-        # CPU quota limits parallel admission, not forward progress. A host
-        # with a fractional CPU must still be able to run one snapshot.
+        # CPU quota bounds parallel work; measured memory admission is done
+        # only after the first complete snapshot establishes its actual cost.
         return max(1, min(configured, cpus))
 
+    def additional_snapshot_slots(self, configured: int, active: int, observed_cost: int | None) -> int:
+        if not observed_cost:
+            # Stage one complete real capture before spending memory on parallel
+            # hook trees. A zero-cost/failed probe cannot justify fanout.
+            return 1 if active == 0 and configured > 0 else 0
+        headroom = self.memory_headroom()
+        return self.slots_for_headroom(configured, active, observed_cost, headroom[1] if headroom else 0)
+
+    @staticmethod
+    def slots_for_headroom(configured: int, active: int, observed_cost: int, available: int) -> int:
+        slots = max(0, configured - active)
+        if observed_cost > 0:
+            # Active captures can still grow to the measured peak. Reserve for
+            # that growth before promising the remaining headroom to new work.
+            unpromised = max(0, available - active * observed_cost)
+            slots = min(slots, unpromised // observed_cost)
+        return max(1, slots) if active == 0 and configured else slots
+
+    def memory_headroom(self) -> tuple[int, int, int] | None:
+        """Return workload RAM+swap usage, effective headroom, and host headroom."""
+        try:
+            meminfo = {
+                name: int(value.split()[0]) * 1024
+                for name, value in (line.split(":", 1) for line in Path("/proc/meminfo").read_text().splitlines() if ":" in line)
+            }
+            host_swap_free = host_swap_free_bytes(meminfo.get("SwapFree", 0))
+            host_available = meminfo["MemAvailable"] + host_swap_free
+        except (OSError, ValueError, KeyError):
+            host_swap_free = 0 if sys.platform.startswith("linux") else psutil.swap_memory().free
+            host_available = psutil.virtual_memory().available + host_swap_free
+
+        available = host_available
+        used_bytes = None
+        for path in self.cgroup_paths():
+            try:
+                memory_max = (path / "memory.max").read_text().strip()
+                if memory_max == "max":
+                    continue
+                current = int((path / "memory.current").read_text())
+                swap_max = (path / "memory.swap.max").read_text().strip()
+                swap_current = int((path / "memory.swap.current").read_text())
+                swap_room = host_swap_free if swap_max == "max" else min(host_swap_free, max(0, int(swap_max) - swap_current))
+                available = min(available, max(0, int(memory_max) - current) + swap_room)
+                if used_bytes is None:
+                    used_bytes = current + swap_current
+            except (OSError, ValueError, KeyError):
+                continue
+        if used_bytes is None:
+            # Bare-metal and non-Linux installs have no finite cgroup. Count
+            # the real runner process tree rather than the entire host.
+            try:
+                process = psutil.Process()
+                processes = [process, *process.children(recursive=True)]
+            except (psutil.Error, OSError):
+                return None
+            used_bytes = 0
+            for child in processes:
+                try:
+                    used_bytes += child.memory_info().rss
+                except (psutil.Error, OSError):
+                    continue
+        return used_bytes, available, host_available
+
     def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
         previous = self._memory_stalls.get(source)
         self._memory_stalls[source] = (total, sampled_at)

archivebox/services/runner.py

@@ -232,6 +232,10 @@ class CrawlRunner:
         self.snapshot_tasks: dict[str, asyncio.Task[None]] = {}
         self.snapshot_semaphore = asyncio.Semaphore(1)
         self.max_concurrent_snapshots = 1
+        self._warmup_snapshot_id: str | None = None
+        self._memory_baseline: tuple[int, int] | None = None
+        self._memory_peak_cost = 0
+        self._observed_snapshot_cost: int | None = None
         self._resource_deferred = False
         self._initial_snapshot_admitted = False
         self.persona = None
@@ -498,10 +502,15 @@ class CrawlRunner:
         if task is not None and not task.done():
             return
         current_event = crawl_start_event or get_current_event()
+        if not isinstance(current_event, CrawlStartEvent) and in_handler_context():
+            return
+        if self._observed_snapshot_cost is None and self._warmup_snapshot_id is None:
+            observation = resource_admission.memory_headroom()
+            self._memory_baseline = (observation[0], observation[2]) if observation is not None else None
+            self._memory_peak_cost = 0
+            self._warmup_snapshot_id = snapshot_id
         if isinstance(current_event, CrawlStartEvent):
             task = asyncio.create_task(self.run_snapshot(snapshot_id, current_event), context=_runner_task_context())
-        elif in_handler_context():
-            return
         else:
             task = asyncio.create_task(self.run_snapshot(snapshot_id), context=_runner_task_context())
         self._initial_snapshot_admitted = False
@@ -530,6 +539,7 @@ class CrawlRunner:
                 if task.done():
                     if self.snapshot_tasks.get(snapshot_id) is task:
                         self.snapshot_tasks.pop(snapshot_id, None)
+                    self._finish_warmup_snapshot(snapshot_id)
                     try:
                         task.result()
                     except asyncio.CancelledError as err:
@@ -554,13 +564,16 @@ class CrawlRunner:
                     return
                 continue
             await self.heartbeat_active_leases()
-            done, _pending = await asyncio.wait(pending_tasks, timeout=10.0, return_when=asyncio.FIRST_COMPLETED)
+            self._observe_snapshot_memory()
+            done, _pending = await asyncio.wait(pending_tasks, timeout=RESOURCE_RECHECK_SECONDS, return_when=asyncio.FIRST_COMPLETED)
+            self._observe_snapshot_memory()
             if not done:
                 continue
             for task in done:
                 for snapshot_id, tracked_task in list(self.snapshot_tasks.items()):
                     if tracked_task is task:
                         self.snapshot_tasks.pop(snapshot_id, None)
+                        self._finish_warmup_snapshot(snapshot_id)
                         break
                 try:
                     task.result()
@@ -578,6 +591,36 @@ class CrawlRunner:
             if not stop_scheduling and enqueue_projected:
                 await self.enqueue_pending_snapshots_from_projection()
 
+    def _observe_snapshot_memory(self) -> None:
+        if self._memory_baseline is None:
+            return
+        observation = resource_admission.memory_headroom()
+        if observation is not None:
+            used, _available, host_available = observation
+            baseline_used, baseline_host_available = self._memory_baseline
+            # Host headroom sees helpers outside ArchiveBox's cgroup. Keep a
+            # high-water cost as later URLs may use more than the first one.
+            # Never divide concurrent growth by task count: their pages vary.
+            self._memory_peak_cost = max(
+                self._memory_peak_cost,
+                used - baseline_used,
+                baseline_host_available - host_available,
+            )
+            if self._observed_snapshot_cost is not None:
+                self._observed_snapshot_cost = max(self._observed_snapshot_cost, self._memory_peak_cost)
+
+    def _finish_warmup_snapshot(self, snapshot_id: str) -> None:
+        if snapshot_id != self._warmup_snapshot_id or self._observed_snapshot_cost is not None:
+            return
+        self._observe_snapshot_memory()
+        if self._memory_peak_cost > 0:
+            self._observed_snapshot_cost = self._memory_peak_cost
+        else:
+            # No observed cost cannot justify parallel admission. Probe the
+            # next real snapshot while the queue keeps moving one at a time.
+            self._memory_baseline = None
+        self._warmup_snapshot_id = None
+
     async def heartbeat_active_leases(self) -> None:
         # These are resumable work-item leases, not orchestrator-election
         # heartbeats. Each update is a short autocommit statement; network and
@@ -645,7 +688,11 @@ class CrawlRunner:
         self.max_concurrent_snapshots = resource_admission.snapshot_limit(int(config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]))
 
         active_snapshot_ids = [snapshot_id for snapshot_id, task in self.snapshot_tasks.items() if not task.done()]
-        available_slots = max(0, self.max_concurrent_snapshots - len(active_snapshot_ids))
+        available_slots = resource_admission.additional_snapshot_slots(
+            self.max_concurrent_snapshots,
+            len(active_snapshot_ids),
+            self._observed_snapshot_cost,
+        )
         if available_slots <= 0:
             return
         pending_snapshot_ids = await sync_to_async(
@@ -997,7 +1044,8 @@ class CrawlRunner:
             if self.initial_snapshot_ids is not None:
                 remaining_snapshot_ids = iter(snapshot_ids)
                 while True:
-                    batch = list(zip(range(self.max_concurrent_snapshots), remaining_snapshot_ids, strict=False))
+                    slots = resource_admission.additional_snapshot_slots(self.max_concurrent_snapshots, 0, self._observed_snapshot_cost)
+                    batch = list(zip(range(slots), remaining_snapshot_ids, strict=False))
                     if not batch:
                         return
                     for _slot, snapshot_id in batch:
@@ -1008,7 +1056,8 @@ class CrawlRunner:
                         await self.enqueue_snapshot(snapshot_id)
                     await self.wait_for_snapshot_tasks(enqueue_projected=False)
             else:
-                for snapshot_id in snapshot_ids[: self.max_concurrent_snapshots]:
+                slots = resource_admission.additional_snapshot_slots(self.max_concurrent_snapshots, 0, self._observed_snapshot_cost)
+                for snapshot_id in snapshot_ids[:slots]:
                     if await self.crawl_is_cancelled():
                         break
                     if await self.crawl_is_paused():
@@ -1471,7 +1520,9 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
             # avoids hot-looping on the parent while child work is ready without
             # resurrecting a user cancellation that sealed the crawl after
             # selection.
-            crawl.safe_update(
+            if crawl.status != crawl.StatusChoices.STARTED:
+                return False
+            return crawl.safe_update(
                 {
                     "status": crawl.StatusChoices.STARTED,
                     "retry_at": now + timedelta(seconds=ACTIVE_STATE_LEASE_SECONDS),
@@ -1480,7 +1531,6 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
                 refresh=False,
                 extra_filter={"status": crawl.StatusChoices.STARTED},
             )
-            return True
         if snapshot_count and not due_active_snapshots:
             if crawl.is_finished():
                 if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
@@ -1838,12 +1888,11 @@ def _run_due_crawl_status(
     due_crawl = Crawl.objects.filter(id=due_crawl_id).first()
     if due_crawl is None:
         return True
-    run_due_crawl(
+    return run_due_crawl(
         due_crawl,
         lock_seconds=lock_seconds,
         interactive_interrupts=interactive_interrupts,
     )
-    return True
 
 
 def _run_due_snapshot_query(
@@ -1897,8 +1946,7 @@ def _run_due_binary() -> bool:
     due_binary = Binary.objects.filter(id=due_binary_id).first()
     if due_binary is None:
         return True
-    run_due_binary(due_binary, lock_seconds=60)
-    return True
+    return run_due_binary(due_binary, lock_seconds=60)
 
 
 def run_pending_crawls(

archivebox/tests/test_migrations_08_to_09.py

@@ -12,12 +12,18 @@ Migration tests from 0.8.x to 0.9.x.
 
 import sqlite3
 import json
+import signal
+import subprocess
+import time
 import uuid
 from datetime import datetime
 from urllib.parse import urlparse
 
+import psutil
 import pytest
 
+from archivebox.tests.conftest import cli_env, run_archivebox_cmd, stop_archivebox_process
+
 from .migrations_helpers import (
     SCHEMA_0_7,
     SCHEMA_0_8,
@@ -225,6 +231,118 @@ def test_migration_preserves_crawls(migration_08_data):
     assert ok, msg
 
 
+def test_legacy_default_crawl_is_sealed_without_a_due_retry(tmp_path):
+    """Upgrading a legacy collection must not leave its sealed crawl runnable."""
+    work_dir = tmp_path
+    db_path = work_dir / "index.sqlite3"
+    create_data_dir_structure(work_dir)
+    with sqlite3.connect(db_path) as conn:
+        conn.executescript(SCHEMA_0_7)
+    seed_0_7_data(db_path)
+
+    result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=90)
+    assert result.returncode == 0, result.stderr or result.stdout
+
+    with sqlite3.connect(db_path) as conn:
+        rows = conn.execute("SELECT status, retry_at FROM crawls_crawl").fetchall()
+        snapshot_statuses = conn.execute("SELECT DISTINCT status FROM core_snapshot").fetchall()
+        result_statuses = conn.execute("SELECT DISTINCT status FROM core_archiveresult").fetchall()
+    assert rows == [("sealed", None)]
+    assert snapshot_statuses == [("sealed",)]
+    assert set(result_statuses) == {("succeeded",), ("failed",), ("skipped",)}
+
+
+def test_old_migrated_crawl_stays_idle_and_forward_migration_repairs_it(tmp_path):
+    """A dated raw ISO retry from an already-upgraded collection must not spin."""
+    work_dir = tmp_path
+    db_path = work_dir / "index.sqlite3"
+    create_data_dir_structure(work_dir)
+    with sqlite3.connect(db_path) as conn:
+        conn.executescript(SCHEMA_0_7)
+    seed_0_7_data(db_path)
+
+    before_repair = run_archivebox_migration_cmd(work_dir, ["manage", "migrate", "core", "0054", "--noinput"], timeout=90)
+    assert before_repair.returncode == 0, before_repair.stderr or before_repair.stdout
+    with sqlite3.connect(db_path) as conn:
+        # 0024 used datetime.now().isoformat() for this sealed Crawl. Represent
+        # a collection upgraded before today so SQLite's lexical due query sees it.
+        conn.execute("UPDATE crawls_crawl SET retry_at = '2024-01-01T00:00:00.123456'")
+        assert conn.execute("SELECT status, retry_at FROM crawls_crawl").fetchall() == [
+            ("sealed", "2024-01-01T00:00:00.123456"),
+        ]
+        assert conn.execute("SELECT DISTINCT status FROM core_snapshot").fetchall() == [("sealed",)]
+
+    env = cli_env(disable_extractors=True)
+    log_path = work_dir / "idle-daemon.log"
+    with log_path.open("w") as log:
+        daemon = run_archivebox_cmd(
+            ["run", "--daemon"],
+            cwd=work_dir,
+            env=env,
+            stdout=log,
+            stderr=subprocess.STDOUT,
+            capture_output=False,
+            wait=False,
+            start_new_session=True,
+        )
+    try:
+        ready_deadline = time.monotonic() + 20
+        while True:
+            with sqlite3.connect(db_path) as conn:
+                ready = conn.execute(
+                    "SELECT 1 FROM machine_process WHERE process_type = 'orchestrator' AND status = 'running' AND pid = ?",
+                    (daemon.pid,),
+                ).fetchone()
+            if ready:
+                break
+            assert daemon.poll() is None, log_path.read_text()
+            assert time.monotonic() < ready_deadline, log_path.read_text()
+            time.sleep(0.1)
+        cpu_percent = psutil.Process(daemon.pid).cpu_percent(interval=5)
+        assert cpu_percent < 30, f"idle daemon used {cpu_percent}% CPU: {log_path.read_text()}"
+    finally:
+        stop_archivebox_process(daemon, signal.SIGTERM)
+
+    repaired = run_archivebox_migration_cmd(work_dir, ["init"], timeout=90)
+    assert repaired.returncode == 0, repaired.stderr or repaired.stdout
+    with sqlite3.connect(db_path) as conn:
+        assert conn.execute("SELECT status, retry_at FROM crawls_crawl").fetchall() == [("sealed", None)]
+
+
+def test_forward_migration_normalizes_active_legacy_retry_timestamps(migration_08_data):
+    """Preserve instants for active rows while leaving canonical timestamps intact."""
+    work_dir, db_path, _original_data = migration_08_data
+    before_repair = run_archivebox_migration_cmd(work_dir, ["manage", "migrate", "core", "0054", "--noinput"], timeout=90)
+    assert before_repair.returncode == 0, before_repair.stderr or before_repair.stdout
+
+    old_offset_timestamp = "2024-01-01T03:04:05.123456+03:00"
+    expected_utc_timestamp = "2024-01-01 00:04:05.123456"
+    canonical_timestamp = "2025-04-05 06:07:08.123456"
+    with sqlite3.connect(db_path) as conn:
+        crawl_ids = [row[0] for row in conn.execute("SELECT id FROM crawls_crawl WHERE status = 'queued' ORDER BY id")]
+        snapshot_ids = [row[0] for row in conn.execute("SELECT id FROM core_snapshot ORDER BY id")]
+        binary_id = conn.execute("SELECT id FROM machine_binary ORDER BY id LIMIT 1").fetchone()[0]
+        process_id = conn.execute("SELECT id FROM machine_process ORDER BY id LIMIT 1").fetchone()[0]
+        assert len(crawl_ids) == 2
+        assert len(snapshot_ids) >= 2
+        conn.execute("UPDATE crawls_crawl SET retry_at = ? WHERE id = ?", (old_offset_timestamp, crawl_ids[0]))
+        conn.execute("UPDATE crawls_crawl SET retry_at = ? WHERE id = ?", (canonical_timestamp, crawl_ids[1]))
+        conn.execute("UPDATE core_snapshot SET retry_at = ? WHERE id = ?", (old_offset_timestamp, snapshot_ids[0]))
+        conn.execute("UPDATE core_snapshot SET retry_at = ? WHERE id = ?", (canonical_timestamp, snapshot_ids[1]))
+        conn.execute("UPDATE machine_binary SET retry_at = ? WHERE id = ?", (old_offset_timestamp, binary_id))
+        conn.execute("UPDATE machine_process SET retry_at = ? WHERE id = ?", (old_offset_timestamp, process_id))
+
+    repaired = run_archivebox_migration_cmd(work_dir, ["init"], timeout=90)
+    assert repaired.returncode == 0, repaired.stderr or repaired.stdout
+    with sqlite3.connect(db_path) as conn:
+        assert conn.execute("SELECT retry_at FROM crawls_crawl WHERE id = ?", (crawl_ids[0],)).fetchone() == (expected_utc_timestamp,)
+        assert conn.execute("SELECT retry_at FROM crawls_crawl WHERE id = ?", (crawl_ids[1],)).fetchone() == (canonical_timestamp,)
+        assert conn.execute("SELECT retry_at FROM core_snapshot WHERE id = ?", (snapshot_ids[0],)).fetchone() == (expected_utc_timestamp,)
+        assert conn.execute("SELECT retry_at FROM core_snapshot WHERE id = ?", (snapshot_ids[1],)).fetchone() == (canonical_timestamp,)
+        assert conn.execute("SELECT retry_at FROM machine_binary WHERE id = ?", (binary_id,)).fetchone() == (expected_utc_timestamp,)
+        assert conn.execute("SELECT retry_at FROM machine_process WHERE id = ?", (process_id,)).fetchone() == (expected_utc_timestamp,)
+
+
 def test_migration_preserves_snapshot_crawl_links(migration_08_data):
     """Migration should preserve snapshot-to-crawl relationships and assign default crawl to orphans."""
     work_dir, db_path, original_data = migration_08_data

archivebox/tests/test_recursive_crawl.py

@@ -724,6 +724,60 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
     assert overlapping, f"Expected hook processes from different snapshots to overlap, got intervals: {intervals}"
 
 
+@pytest.mark.timeout(600)
+def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(initialized_archive, recursive_test_site):
+    """A real three-URL capture must not launch all Chrome hook trees at once."""
+    urls = [recursive_test_site["root_url"], *recursive_test_site["child_urls"][:2]]
+    env = os.environ.copy()
+    env.update(
+        {
+            "USE_COLOR": "false",
+            "SHOW_PROGRESS": "false",
+            "URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
+            "ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
+            "CHROME_HEADLESS": "true",
+            "CHROME_SANDBOX": "false",
+            "CHROME_ISOLATION": "crawl",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
+            "TIMEOUT": "90",
+        },
+    )
+    result = run_archivebox_cmd(
+        ["add", "--parser=url_list", "--plugins=chrome,wget,screenshot", "--tag=measured-admission", *urls],
+        cwd=initialized_archive,
+        env=env,
+        timeout=600,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+
+    with use_archivebox_db(initialized_archive):
+        crawl = Crawl.objects.get(tags_str="measured-admission")
+        snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("created_at"))
+        hook_intervals = list(
+            Process.objects.filter(
+                process_type=Process.TypeChoices.HOOK,
+                archiveresult__snapshot__crawl=crawl,
+            )
+            .values_list("archiveresult__snapshot_id", "started_at", "ended_at")
+            .distinct(),
+        )
+
+    assert len(snapshots) == 3
+    assert all(snapshot.status == Snapshot.StatusChoices.SEALED and snapshot.downloaded_at for snapshot in snapshots)
+    first_id = snapshots[0].id
+    first_end = snapshots[0].downloaded_at
+    later_starts = [started for snapshot_id, started, _ended in hook_intervals if snapshot_id != first_id and started]
+    assert later_starts and first_end <= min(later_starts), hook_intervals
+    later_intervals = [
+        (snapshot_id, started, ended) for snapshot_id, started, ended in hook_intervals if snapshot_id != first_id and started and ended
+    ]
+    assert any(
+        left_id != right_id and left_start < right_end and right_start < left_end
+        for index, (left_id, left_start, left_end) in enumerate(later_intervals)
+        for right_id, right_start, right_end in later_intervals[index + 1 :]
+    ), later_intervals
+
+
 def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, initialized_archive):
     """Test that Snapshot model has parent_snapshot field."""
 

archivebox/tests/test_release_source_fingerprint.py

@@ -0,0 +1,174 @@
+import json
+import importlib.util
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+import pytest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+FINGERPRINT_SCRIPT = REPO_ROOT / "bin" / "release-source-fingerprint.py"
+_SPEC = importlib.util.spec_from_file_location("release_source_fingerprint", FINGERPRINT_SCRIPT)
+assert _SPEC is not None and _SPEC.loader is not None
+_FINGERPRINT_MODULE = importlib.util.module_from_spec(_SPEC)
+_SPEC.loader.exec_module(_FINGERPRINT_MODULE)
+
+
+def _git(repo: Path, *args: str) -> str:
+    return subprocess.run(
+        ["git", "-C", str(repo), *args],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+
+
+def _commit(repo: Path, message: str) -> str:
+    _git(repo, "add", "-A")
+    _git(repo, "commit", "-m", message)
+    return _git(repo, "rev-parse", "HEAD")
+
+
+def _fingerprint(repo: Path, revision: str) -> str:
+    return subprocess.run(
+        [sys.executable, str(FINGERPRINT_SCRIPT), revision],
+        cwd=repo,
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+
+
+def _write_template_tree(repo: Path) -> None:
+    (repo / "etc").mkdir()
+    project = (REPO_ROOT / "pyproject.toml").read_text()
+    project = re.sub(r'^version = "[^"]+"$', 'version = "0.9.56rc41"', project, count=1, flags=re.M)
+    project = re.sub(r'^current_version = "[^"]+"$', 'current_version = "v0.9.56rc41"', project, count=1, flags=re.M)
+    assert 'version = "0.9.56rc41"' in project
+    assert 'current_version = "v0.9.56rc41"' in project
+    lock, count = re.subn(
+        r'(\[\[package\]\]\nname = "archivebox"\nversion = ")[^"]+',
+        r"\g<1>0.9.56rc41",
+        (REPO_ROOT / "uv.lock").read_text(),
+        count=1,
+    )
+    assert count == 1
+    package = json.loads((REPO_ROOT / "etc/package.json").read_text())
+    package["version"] = "0.9.56rc41"
+    (repo / "pyproject.toml").write_text(project)
+    (repo / "uv.lock").write_text(lock)
+    (repo / "etc/package.json").write_text(json.dumps(package, indent=2) + "\n")
+    (repo / "source.py").write_text("release behavior = unchanged\n")
+
+
+def _new_repo(tmp_path: Path) -> tuple[Path, str]:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    subprocess.run(["git", "init", "-q", str(repo)], check=True)
+    _git(repo, "config", "user.name", "ArchiveBox Tests")
+    _git(repo, "config", "user.email", "tests@archivebox.io")
+    _write_template_tree(repo)
+    return repo, _commit(repo, "stable acceptance source")
+
+
+def test_fingerprint_ignores_only_archivebox_release_version_fields(tmp_path):
+    repo, accepted = _new_repo(tmp_path)
+    project = repo / "pyproject.toml"
+    text = project.read_text()
+    text = re.sub(r'^version = "[^"]+"$', 'version = "0.9.64"', text, count=1, flags=re.M)
+    text = re.sub(r'^current_version = "[^"]+"$', 'current_version = "v0.9.64"', text, count=1, flags=re.M)
+    project.write_text(text)
+    lock = repo / "uv.lock"
+    lock.write_text(re.sub(r'(?m)^(version = ")0\.9\.56rc41("\s*)$', r"\g<1>0.9.64\2", lock.read_text(), count=1))
+    package = repo / "etc/package.json"
+    package.write_text(package.read_text().replace('"version": "0.9.56rc41"', '"version": "0.9.64"'))
+    stable = _commit(repo, "stable version bump")
+
+    assert _fingerprint(repo, accepted) == _fingerprint(repo, stable)
+
+
+def test_pyproject_version_normalization_matches_exact_template_bytes():
+    project = (REPO_ROOT / "pyproject.toml").read_bytes()
+    normalized = _FINGERPRINT_MODULE.normalize_file(b"pyproject.toml", project)
+    expected = re.sub(
+        rb'(?m)^(version = ")[^"\r\n]+("\s*(?:#.*)?\r?\n?)$',
+        lambda match: match.group(1) + b"<ARCHIVEBOX_PROJECT_VERSION>" + match.group(2),
+        project,
+        count=1,
+    )
+    expected = re.sub(
+        rb'(?m)^(current_version = ")[^"\r\n]+("\s*(?:#.*)?\r?\n?)$',
+        lambda match: match.group(1) + b"<ARCHIVEBOX_PROJECT_VERSION>" + match.group(2),
+        expected,
+        count=1,
+    )
+    assert normalized == expected
+
+
+@pytest.mark.parametrize(
+    "change",
+    [
+        "source",
+        "project_dependency",
+        "lock_dependency",
+        "package_dependency",
+        "other_tool_version",
+        "file_mode",
+    ],
+)
+def test_fingerprint_detects_non_release_version_changes(tmp_path, change):
+    repo, accepted = _new_repo(tmp_path)
+    if change == "source":
+        (repo / "source.py").write_text("release behavior = changed\n")
+    elif change == "project_dependency":
+        path = repo / "pyproject.toml"
+        project = path.read_text()
+        project, count = re.subn(r'("abx-dl==)[^"]+(")', r"\g<1>999.0\2", project, count=1)
+        assert count == 1
+        path.write_text(project)
+    elif change == "lock_dependency":
+        path = repo / "uv.lock"
+        path.write_text(re.sub(r'(?m)(^name = "abx-dl"\nversion = ")[^"]+', r"\g<1>999.0", path.read_text(), count=1))
+    elif change == "package_dependency":
+        path = repo / "etc/package.json"
+        payload = json.loads(path.read_text())
+        name = next(iter(payload["dependencies"]))
+        payload["dependencies"][name] = "999.0"
+        path.write_text(json.dumps(payload, indent=2) + "\n")
+    elif change == "other_tool_version":
+        path = repo / "pyproject.toml"
+        path.write_text(path.read_text() + '\n[tool.other]\nversion = "1.0"\n')
+        _commit(repo, "add unrelated tool version")
+        path.write_text(path.read_text().replace('version = "1.0"', 'version = "2.0"'))
+    elif change == "file_mode":
+        path = repo / "source.py"
+        path.chmod(0o755)
+
+    changed = _commit(repo, change)
+    assert _fingerprint(repo, accepted) != _fingerprint(repo, changed)
+
+
+def test_fingerprint_detects_submodule_gitlink_changes(tmp_path):
+    repo, accepted = _new_repo(tmp_path)
+    subrepo = tmp_path / "dependency"
+    subrepo.mkdir()
+    subprocess.run(["git", "init", "-q", str(subrepo)], check=True)
+    _git(subrepo, "config", "user.name", "ArchiveBox Tests")
+    _git(subrepo, "config", "user.email", "tests@archivebox.io")
+    (subrepo / "dependency.txt").write_text("one\n")
+    first = _commit(subrepo, "first dependency revision")
+    (repo / "vendor").mkdir()
+    _git(repo, "update-index", "--add", "--cacheinfo", f"160000,{first},vendor/dependency")
+    _git(repo, "commit", "-m", "add dependency gitlink")
+    one = _git(repo, "rev-parse", "HEAD")
+
+    (subrepo / "dependency.txt").write_text("two\n")
+    second = _commit(subrepo, "second dependency revision")
+    _git(repo, "update-index", "--cacheinfo", f"160000,{second},vendor/dependency")
+    _git(repo, "commit", "-m", "update dependency gitlink")
+    two = _git(repo, "rev-parse", "HEAD")
+
+    assert _fingerprint(repo, accepted) != _fingerprint(repo, one)
+    assert _fingerprint(repo, one) != _fingerprint(repo, two)

archivebox/tests/test_release_workflow.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 import json
+import importlib.util
 import shlex
 import os
 import re
@@ -13,6 +14,12 @@ REPO_ROOT = Path(__file__).resolve().parents[2]
 RELEASE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release.yml"
 RELEASE_CANDIDATE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release-candidate.yml"
 PIP_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pip.yml"
+DOCKER_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "docker.yml"
+VERIFY_STAGING_SCRIPT = REPO_ROOT / "bin" / "verify-staging-release.py"
+_VERIFY_SPEC = importlib.util.spec_from_file_location("verify_staging_release", VERIFY_STAGING_SCRIPT)
+assert _VERIFY_SPEC is not None and _VERIFY_SPEC.loader is not None
+_VERIFY_STAGING = importlib.util.module_from_spec(_VERIFY_SPEC)
+_VERIFY_SPEC.loader.exec_module(_VERIFY_STAGING)
 
 
 def _git(repo, *args):
@@ -168,6 +175,7 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
     assert python_release["environment"] == "pypi"
     checkout = python_release["steps"][0]
     assert checkout["with"]["token"] == "${{ secrets.RELEASE_GH_TOKEN || github.token }}"
+
     assert docker_release["needs"] == ["candidate", "python-release"]
     assert docker_release["env"]["DOCKER_DIGEST_RUN_ID"] == "${{ needs.candidate.outputs.digest_run_id }}"
     assert "release_ready" not in docker_release["if"]
@@ -245,3 +253,52 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
     create_release_line = logical_lines.index(next(line for line in logical_lines if line.startswith("$GH_BINARY release create ")))
     upload_release_line = logical_lines.index(next(line for line in logical_lines if line.startswith("$GH_BINARY release upload ")))
     assert publish_line < rc_skip_line < create_release_line < upload_release_line
+
+
+def test_stable_publication_requires_live_acceptance_before_upload():
+    workflow = yaml.safe_load(RELEASE_WORKFLOW.read_text())
+    publisher = workflow["jobs"]["python-release"]
+    steps = publisher["steps"]
+    gate = next(i for i, step in enumerate(steps) if step.get("name") == "Require matching acceptance on Cabbage and DigestBox")
+    upload = next(i for i, step in enumerate(steps) if step.get("name") == "Publish the exact tested distributions")
+    assert gate < upload
+    # The same guard applies to automatic and manually requested stable releases.
+    assert steps[gate]["if"] == "env.RELEASE_BRANCH == 'main'"
+    assert "verify-staging-release.py" in steps[gate]["run"]
+    assert not steps[gate].get("continue-on-error")
+    assert publisher["permissions"]["deployments"] == "read"
+
+
+def test_staging_gate_matches_the_exact_tested_docker_base_digest(tmp_path):
+    workflow = yaml.safe_load(RELEASE_WORKFLOW.read_text())
+    publisher = workflow["jobs"]["python-release"]
+    steps = publisher["steps"]
+    metadata = next(step for step in steps if step.get("name") == "Download tested Docker base-image metadata")
+    gate = next(step for step in steps if step.get("name") == "Require matching acceptance on Cabbage and DigestBox")
+    assert metadata["with"]["run-id"] == "${{ env.DOCKER_DIGEST_RUN_ID }}"
+    assert publisher["env"]["DOCKER_DIGEST_RUN_ID"] == "${{ needs.candidate.outputs.digest_run_id }}"
+    assert metadata["with"]["pattern"] == "digest-*"
+    assert metadata["with"]["merge-multiple"] is True
+    assert metadata["with"]["path"] == "${{ runner.temp }}/tested-docker-artifacts"
+    assert steps.index(metadata) < steps.index(gate)
+    assert gate["env"]["STAGING_DOCKER_ARTIFACT_DIR"] == "${{ runner.temp }}/tested-docker-artifacts"
+
+    docker_workflow = yaml.safe_load(DOCKER_WORKFLOW.read_text())
+    export = next(step for step in docker_workflow["jobs"]["build"]["steps"] if step.get("name") == "Export digest")
+    assert export["env"]["ABX_DL_IMAGE_REF"] == "${{ steps.abx_dl_image.outputs.image }}"
+    assert '"/tmp/digests/abx-dl-image-$ARTIFACT_NAME"' in export["run"]
+    helper = (REPO_ROOT / "bin" / "staging-acceptance-host.sh").read_text()
+    assert "abx_dl_image" in helper
+
+    source_sha = "a" * 40
+    base_digest = "sha256:" + "b" * 64
+    (tmp_path / "source-ref").write_text(source_sha + "\n")
+    for platform in ("linux-amd64", "linux-arm64"):
+        (tmp_path / f"abx-dl-image-digest-{platform}").write_text(f"archivebox/abx-dl:1.13.51@{base_digest}\n")
+    assert _VERIFY_STAGING.tested_abx_dl_digest(tmp_path, source_sha) == base_digest
+
+    (tmp_path / "abx-dl-image-digest-linux-arm64").write_text(f"archivebox/abx-dl:1.13.52@{'sha256:' + 'c' * 64}\n")
+    with pytest.raises(SystemExit, match="different abx-dl images"):
+        _VERIFY_STAGING.tested_abx_dl_digest(tmp_path, source_sha)
+    with pytest.raises(SystemExit, match="different source commit"):
+        _VERIFY_STAGING.tested_abx_dl_digest(tmp_path, "d" * 40)

archivebox/tests/test_resource_admission.py

@@ -1,6 +1,7 @@
 import os
+import subprocess
 
-from archivebox.services.resource_admission import ResourceAdmission
+from archivebox.services.resource_admission import ResourceAdmission, disk_backed_swap_free_bytes
 
 
 def test_cpu_admission_respects_process_affinity():
@@ -10,6 +11,69 @@ def test_cpu_admission_respects_process_affinity():
     assert admission.snapshot_limit(1) == 1
 
 
+def test_snapshot_admission_stages_first_capture_then_uses_measured_headroom():
+    admission = ResourceAdmission()
+    assert admission.additional_snapshot_slots(3, active=0, observed_cost=None) == 1
+    assert admission.additional_snapshot_slots(3, active=1, observed_cost=None) == 0
+    # One observed peak costs 500 MB; only measured space is reserved for
+    # each additional in-flight snapshot, with no fixed minimum RAM.
+    assert admission.slots_for_headroom(3, active=0, observed_cost=500_000_000, available=1_100_000_000) == 2
+    assert admission.slots_for_headroom(3, active=1, observed_cost=500_000_000, available=400_000_000) == 0
+    assert admission.slots_for_headroom(3, active=1, observed_cost=500_000_000, available=900_000_000) == 0
+    assert admission.slots_for_headroom(3, active=1, observed_cost=500_000_000, available=1_100_000_000) == 1
+    assert admission.slots_for_headroom(3, active=0, observed_cost=500_000_000, available=0) == 1
+    # Headroom from memory_headroom() also includes disk-backed free swap.
+    assert admission.slots_for_headroom(3, active=0, observed_cost=500_000_000, available=2_000_000_000) == 3
+    assert admission.additional_snapshot_slots(3, active=0, observed_cost=0) == 1
+
+
+def test_memory_observation_tracks_a_real_child_process():
+    admission = ResourceAdmission()
+    before = admission.memory_headroom()
+    assert before is not None
+    child = subprocess.Popen(
+        [
+            "uv",
+            "run",
+            "python",
+            "-c",
+            "import sys; allocation = bytearray(64 * 1024 * 1024); allocation[0] = 1; print(len(allocation), flush=True); sys.stdin.read()",
+        ],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        assert child.stdout is not None
+        assert child.stdout.readline().strip() == str(64 * 1024 * 1024)
+        during = admission.memory_headroom()
+        assert during is not None
+        assert during[0] > before[0] + 32 * 1024 * 1024
+    finally:
+        assert child.stdin is not None
+        child.stdin.close()
+        assert child.wait(timeout=10) == 0
+
+
+def test_disk_backed_swap_headroom_excludes_zram_but_keeps_backing_swap():
+    swaps = """Filename Type Size Used Priority
+/dev/zram0 partition 1000 100 100
+/swapfile file 2000 500 -2
+/dev/disk-swap partition 4000 1000 -3
+"""
+
+    assert disk_backed_swap_free_bytes(swaps, {"zram0", "254:0"}) == 4_500 * 1024
+
+
+def test_disk_backed_swap_headroom_excludes_zram_device_alias():
+    swaps = """Filename Type Size Used Priority
+/dev/block/254:0 partition 800 200 100
+/swapfile file 500 100 -2
+"""
+
+    assert disk_backed_swap_free_bytes(swaps, {"zram0", "254:0"}) == 400 * 1024
+
+
 def test_memory_stall_counter_recovers_without_resetting_monitor():
     # Real kernel PSI units: cumulative microseconds and ten-second average.
     # Decision samples exercise the policy, not simulated hooks/processes.

archivebox/tests/test_snapshot_output_stacks.py

@@ -311,6 +311,43 @@ def test_responses_card_uses_relative_preview_urls_in_static_export(snapshot):
     assert "localhost" not in html
 
 
+@pytest.mark.django_db(transaction=True)
+def test_opening_files_from_queued_snapshot_stops_detached_progress_monitor(snapshot, live_server):
+    from urllib.parse import urlsplit
+
+    from playwright.sync_api import sync_playwright
+
+    from archivebox.machine.models import Machine
+
+    port = urlsplit(live_server.url).port
+    machine = Machine.current()
+    machine.config = {**machine.config, "BASE_URL": f"http://archivebox.localhost:{port}"}
+    machine.save(update_fields=["config"])
+    snapshot.permissions = "public"
+    snapshot.status = "queued"
+    snapshot.save(update_fields=["permissions", "status"])
+    Path(snapshot.output_dir).mkdir(parents=True, exist_ok=True)
+    errors = []
+    progress_requests = []
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
+        page = browser.new_page()
+        page.on("pageerror", lambda error: errors.append(str(error)))
+        page.on("console", lambda message: errors.append(message.text) if message.type == "error" else None)
+        page.on("request", lambda request: progress_requests.append(request.url) if "progress.json" in request.url else None)
+        page.goto(f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}/index.html#all")
+        page.locator("#main-frame").wait_for()
+        assert page.locator("#progress-monitor").count() == 0
+        # Let the real one-second polling interval and any in-flight response
+        # complete: a detached panel must neither throw nor keep making requests.
+        page.wait_for_timeout(2200)
+        count = len(progress_requests)
+        page.wait_for_timeout(2200)
+        assert len(progress_requests) == count
+        assert errors == []
+        browser.close()
+
+
 @pytest.mark.django_db(transaction=True)
 def test_stack_cover_and_expanded_card_load_same_document(snapshot, live_server):
     from urllib.parse import urlsplit

bin/check-staging-deployment.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Exercise the automatically deployed candidate; never deploy it ourselves."""
+
+import json
+import hashlib
+import os
+import re
+import subprocess
+import sys
+import time
+import tomllib
+from pathlib import Path
+
+from playwright.sync_api import sync_playwright
+
+
+def main():
+    target = os.environ["STAGING_TARGET"]
+    sha = os.environ["STAGING_SHA"]
+    host, port = {"cabbage": ("100.66.141.33", "44"), "digestbox": ("146.190.134.33", "22")}[target]
+    evidence = Path("staging-evidence")
+    evidence.mkdir(exist_ok=True)
+    ssh = [
+        "ssh",
+        "-F",
+        "/dev/null",
+        "-o",
+        "BatchMode=yes",
+        "-o",
+        "IdentitiesOnly=yes",
+        "-o",
+        "StrictHostKeyChecking=yes",
+        "-o",
+        f"UserKnownHostsFile={os.environ['STAGING_KNOWN_HOSTS_FILE']}",
+        "-o",
+        "ConnectTimeout=15",
+        "-i",
+        os.environ["STAGING_KEY_FILE"],
+        "-p",
+        port,
+        f"root@{host}",
+    ]
+    deadline = time.monotonic() + 90 * 60
+    inspection = None
+    expected_pins = {
+        p["name"]: p["version"]
+        for p in tomllib.loads(Path("uv.lock").read_text())["package"]
+        if p["name"] in {"abx-dl", "abx-plugins", "abxbus", "abxpkg"}
+    }
+
+    def remote(operation):
+        return subprocess.run([*ssh, f"{operation} {sha}"], text=True, capture_output=True, timeout=180)
+
+    while time.monotonic() < deadline:
+        check = remote("inspect")
+        if check.returncode == 75 and inspection is None:
+            # Deployment is asynchronous; this is observation, not a test retry
+            # or permission to pull/restart a container from CI.
+            print(check.stderr.strip(), flush=True)
+            time.sleep(30)
+            continue
+        if check.returncode:
+            raise RuntimeError(check.stderr)
+        current = json.loads(check.stdout)
+        if current["helper_sha256"] != hashlib.sha256(Path("bin/staging-acceptance-host.sh").read_bytes()).hexdigest():
+            raise RuntimeError("Staging host acceptance helper differs from the reviewed candidate")
+        if not isinstance(current.get("abx_dl_image"), str) or not re.search(r"@sha256:[0-9a-f]{64}$", current["abx_dl_image"]):
+            raise RuntimeError("Staging host did not report a digest-pinned abx-dl base image")
+        if inspection is not None and current != inspection:
+            raise RuntimeError("The deployed container changed during acceptance")
+        inspection = current
+        result = remote("capture")
+        (evidence / "capture.log").write_text(result.stdout + result.stderr)
+        if result.returncode:
+            raise RuntimeError(result.stderr or result.stdout)
+        report = json.loads(
+            next(line.removeprefix("STAGING_REPORT=") for line in result.stdout.splitlines() if line.startswith("STAGING_REPORT=")),
+        )
+        (evidence / "capture.json").write_text(json.dumps(report, indent=2))
+        if report["pins"] != expected_pins:
+            raise RuntimeError(f"Deployed dependency pins differ: {report['pins']} != {expected_pins}")
+        print(f"{target}: {report['status']}, crawl {report['crawl_id']}", flush=True)
+        if report["status"] == "failed":
+            raise RuntimeError("\n".join(report["errors"]))
+        if report["status"] == "passed":
+            break
+        time.sleep(30)
+    else:
+        raise RuntimeError("Automatic deployment/capture did not complete within the acceptance window")
+
+    with sync_playwright() as playwright:
+        browser = playwright.chromium.launch()
+        page = browser.new_page(viewport={"width": 1440, "height": 1000})
+        for snapshot in report["snapshots"]:
+            response = page.goto(snapshot["replay_url"], wait_until="domcontentloaded")
+            assert response and response.ok, snapshot["replay_url"]
+            page.locator('.thumb-card[data-plugin-name="screenshot"]').wait_for(state="attached")
+            page.locator("#main-frame").wait_for(state="attached")
+            page.screenshot(path=str(evidence / f"{snapshot['id']}.png"), full_page=True)
+        browser.close()
+    # Recheck identity and host OOM history after the browser flow too.
+    final = remote("capture")
+    if final.returncode:
+        raise RuntimeError(final.stderr)
+    final_report = json.loads(
+        next(line.removeprefix("STAGING_REPORT=") for line in final.stdout.splitlines() if line.startswith("STAGING_REPORT=")),
+    )
+    final_inspection = remote("inspect")
+    if final_report != report or final_inspection.returncode or json.loads(final_inspection.stdout) != inspection:
+        raise RuntimeError("Deployment or capture evidence changed during the browser acceptance flow")
+    fingerprint = subprocess.check_output([sys.executable, "bin/release-source-fingerprint.py", sha], text=True).strip()
+    payload = {"acceptance_version": 1, "source_fingerprint": fingerprint, "run_id": os.environ["GITHUB_RUN_ID"], "inspection": inspection}
+    request = {
+        "ref": sha,
+        "environment": target,
+        "auto_merge": False,
+        "required_contexts": [],
+        "production_environment": False,
+        "payload": payload,
+        "description": "Real deployed capture, output and browser acceptance",
+    }
+    repository = os.environ["GITHUB_REPOSITORY"]
+    deployment = json.loads(
+        subprocess.check_output(["gh", "api", f"repos/{repository}/deployments", "--input", "-"], input=json.dumps(request), text=True),
+    )
+    subprocess.run(
+        ["gh", "api", f"repos/{repository}/deployments/{deployment['id']}/statuses", "--input", "-"],
+        input=json.dumps(
+            {
+                "state": "success",
+                "auto_inactive": False,
+                "log_url": f"https://github.com/{repository}/actions/runs/{os.environ['GITHUB_RUN_ID']}",
+            },
+        ),
+        text=True,
+        check=True,
+    )
+
+
+if __name__ == "__main__":
+    main()

bin/release-source-fingerprint.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Fingerprint a Git tree while ignoring only ArchiveBox's own release version."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import subprocess
+import sys
+
+
+VERSION_MARKER = b"<ARCHIVEBOX_PROJECT_VERSION>"
+
+
+def git(*args: str) -> bytes:
+    return subprocess.run(
+        ["git", *args],
+        check=True,
+        capture_output=True,
+    ).stdout
+
+
+def normalize_file(path: bytes, content: bytes) -> bytes:
+    """Replace exact own-version values while preserving all other file bytes."""
+    if path == b"pyproject.toml":
+        lines = content.splitlines(keepends=True)
+        section = b""
+        replacements = {b"project": 0, b"tool.bumpver": 0}
+        normalized = []
+        for line in lines:
+            stripped = line.strip()
+            if stripped.startswith(b"[") and stripped.endswith(b"]"):
+                section = stripped[1:-1]
+            key = b"version" if section == b"project" else b"current_version" if section == b"tool.bumpver" else None
+            if key is not None:
+                match = re.fullmatch(rb"(\s*" + key + rb"\s*=\s*)\"[^\"\r\n]+(\"\s*(?:#.*)?\r?\n?)", line)
+                if match:
+                    replacements[section] += 1
+                    line = match.group(1) + b'"' + VERSION_MARKER + match.group(2)
+            normalized.append(line)
+        if replacements != {b"project": 1, b"tool.bumpver": 1}:
+            raise ValueError("pyproject.toml must contain exactly one project.version and tool.bumpver.current_version")
+        return b"".join(normalized)
+
+    if path == b"uv.lock":
+        blocks = re.split(rb"(?m)(?=^\[\[package\]\]\s*$)", content)
+        found = 0
+        for index, block in enumerate(blocks):
+            if not re.search(rb'(?m)^name = "archivebox"\s*$', block):
+                continue
+            updated, count = re.subn(
+                rb'(?m)^(version = ")[^"\r\n]+("\s*)$',
+                lambda match: match.group(1) + VERSION_MARKER + match.group(2),
+                block,
+                count=1,
+            )
+            if count != 1:
+                raise ValueError("uv.lock ArchiveBox package must have exactly one version")
+            blocks[index] = updated
+            found += 1
+        if found != 1:
+            raise ValueError("uv.lock must contain exactly one ArchiveBox package")
+        return b"".join(blocks)
+
+    if path == b"etc/package.json":
+        normalized, count = re.subn(
+            rb'(?m)^(  "version": ")[^"\r\n]+("[,]?\s*)$',
+            lambda match: match.group(1) + VERSION_MARKER + match.group(2),
+            content,
+            count=1,
+        )
+        if count != 1:
+            raise ValueError("etc/package.json must contain exactly one top-level version")
+        return normalized
+
+    return content
+
+
+def fingerprint(revision: str) -> str:
+    entries = git("ls-tree", "-rz", "--full-tree", "-r", revision).split(b"\0")
+    digest = hashlib.sha256()
+    for entry in entries:
+        if not entry:
+            continue
+        metadata, path = entry.split(b"\t", 1)
+        mode, object_type, object_id = metadata.split(b" ")
+        if object_type == b"blob":
+            content = normalize_file(path, git("cat-file", "blob", object_id.decode("ascii")))
+        elif mode == b"160000" and object_type == b"commit":
+            content = object_id
+        else:
+            raise ValueError(f"unsupported Git tree entry: {metadata!r}")
+        for value in (path, mode, object_type, content):
+            digest.update(len(value).to_bytes(8, "big"))
+            digest.update(value)
+    return digest.hexdigest()
+
+
+if __name__ == "__main__":
+    print(fingerprint(sys.argv[1] if len(sys.argv) > 1 else "HEAD"))

bin/staging-acceptance-host.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# Root-owned SSH forced command on our test hosts. Never evaluate the caller's
+# input as shell, accept a URL, or permit deployment/container-control commands.
+set -Eeuo pipefail
+if [[ ! "${SSH_ORIGINAL_COMMAND:-}" =~ ^(inspect|capture)\ ([0-9a-f]{40})$ ]]; then
+    echo 'Only inspect/capture followed by an exact source SHA is allowed.' >&2
+    exit 64
+fi
+operation="${BASH_REMATCH[1]}"
+source_sha="${BASH_REMATCH[2]}"
+case "$(hostname -s)" in
+    cabbage) container=archiveboxdemo-archivebox-1 ;;
+    brussel) container=archivebox-archivebox-1 ;;
+    *) echo 'Not an ArchiveBox staging host.' >&2; exit 64 ;;
+esac
+revision="$(docker inspect "$container" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')"
+if [[ "$revision" != "$source_sha" ]]; then
+    echo "Awaiting automatic deployment: running $revision, expected $source_sha" >&2
+    exit 75
+fi
+test "$(docker inspect "$container" --format '{{.State.Health.Status}}')" = healthy
+# A source bind mount would invalidate the immutable image's provenance.
+while IFS= read -r destination; do
+    case "$destination" in
+        /app|/app/*|/venv|/venv/*|/opt/archivebox|/opt/archivebox/*)
+            echo "Refusing code mount at $destination" >&2; exit 1 ;;
+    esac
+done < <(docker inspect "$container" --format '{{range .Mounts}}{{println .Destination}}{{end}}')
+
+if [[ "$operation" == inspect ]]; then
+    helper_hash="$(sha256sum "$0" | cut -d ' ' -f 1)"
+    docker inspect "$container" --format "{\"helper_sha256\":\"$helper_hash\",\"container_id\":{{json .Id}},\"image_id\":{{json .Image}},\"revision\":{{json (index .Config.Labels \"org.opencontainers.image.revision\")}},\"version\":{{json (index .Config.Labels \"org.opencontainers.image.version\")}},\"abx_dl_image\":{{json (index .Config.Labels \"io.archivebox.abx-dl.image\")}},\"health\":{{json .State.Health.Status}}}"
+    exit 0
+fi
+
+# Persist the start of this candidate's real acceptance flow. Repeated status
+# requests observe the same crawl instead of repeatedly adding test data.
+install -d -m 700 /var/lib/archivebox-staging
+exec 9>/var/lib/archivebox-staging/acceptance.lock
+flock -x 9
+start_file="/var/lib/archivebox-staging/$source_sha.started"
+if [[ ! -e "$start_file" ]]; then date -u +%FT%TZ > "$start_file"; fi
+since="$(cat "$start_file")"
+# Read separately so a journal failure cannot masquerade as "no OOM found"
+# through the conditional pipeline's nonzero exit status.
+kernel_log="$(journalctl -k --since "$since" --no-pager)"
+if grep 'Out of memory: Killed process' <<< "$kernel_log" >/dev/null; then
+    echo 'Staging acceptance failed: the host had an OOM kill during this candidate test.' >&2
+    exit 1
+fi
+
+script="$(cat <<'PY'
+import json
+import os
+from pathlib import Path
+from archivebox.crawls.models import Crawl
+from archivebox.cli.archivebox_add import add
+from archivebox.config import VERSION
+from archivebox.core.routes_util import build_snapshot_detail_url
+from archivebox.plugins.discovery import get_plugin_catalog
+from importlib.metadata import version
+
+sha = os.environ['ACCEPTANCE_SHA']
+tag = f'release-acceptance-{sha}'
+urls = ['https://docs.sweeting.me/s/cookie-dilemma', 'https://sweeting.me/']
+capture_plugins = [plugin for plugin in get_plugin_catalog().values() if any(hook.event == 'Snapshot' for hook in plugin.hooks)]
+capture_config = {'PERMISSIONS': 'public', **{plugin.enabled_key: True for plugin in capture_plugins}}
+crawls = list(Crawl.objects.filter(tags_str=tag).order_by('created_at'))
+if not crawls:
+    # Normal add/runner flow with every snapshot extractor enabled, including
+    # opt-in TLSNotary. Never select a subset or steal the server runner's lease.
+    crawl, _ = add(urls=urls, depth=0, tag=tag, bg=True, persona='ReleaseAcceptance', config=capture_config)
+    crawls = [crawl]
+if len(crawls) != 1:
+    raise RuntimeError('Expected one acceptance crawl for this source')
+crawl = crawls[0]
+snapshots = list(crawl.snapshot_set.all())
+report = {
+    'source_sha': sha, 'version': VERSION, 'crawl_id': str(crawl.pk),
+    'pins': {p: version(p) for p in ('abx-dl', 'abx-plugins', 'abxbus', 'abxpkg')},
+    'status': 'pending', 'snapshots': [], 'errors': [],
+    'enabled_capture_plugins': sorted(plugin.name for plugin in capture_plugins),
+}
+complete = crawl.status == 'sealed' and len(snapshots) == len(urls) and all(s.status == 'sealed' for s in snapshots)
+for snapshot in snapshots:
+    results = list(snapshot.archiveresult_set.all())
+    statuses = {}
+    for result in results:
+        statuses.setdefault(result.plugin, []).append(str(result.status))
+    report['snapshots'].append({'id': str(snapshot.pk), 'url': snapshot.url, 'replay_url': build_snapshot_detail_url(snapshot.archive_path_from_db), 'status': snapshot.status, 'plugins': statuses})
+    if not complete:
+        continue
+    # Real stored outputs, not status labels alone. Missing core capture formats
+    # and failed extractors remain visible failures for investigation.
+    for plugin in ('screenshot', 'pdf', 'singlefile', 'archivewebpage', 'readability', 'defuddle', 'hashes', 'tlsnotary'):
+        if 'succeeded' not in statuses.get(plugin, []):
+            report['errors'].append(f'{snapshot.pk}: {plugin} did not succeed')
+    for plugin in capture_plugins:
+        if plugin.name not in statuses:
+            report['errors'].append(f'{snapshot.pk}: {plugin.name} never reported an output status')
+    for result in results:
+        if result.status == 'failed':
+            report['errors'].append(f'{snapshot.pk}: {result.plugin}: {result.output_str}')
+        if result.status != 'succeeded':
+            continue
+        for relative, metadata in (result.output_files or {}).items():
+            if result.plugin == 'chrome':
+                # Chrome's live session handles are intentionally removed at
+                # crawl teardown; they are not archived replay artifacts.
+                continue
+            base = Path(snapshot.output_dir) if metadata.get('root_relative') else Path(snapshot.output_dir) / result.plugin
+            output = base / relative
+            if not output.is_file() or output.stat().st_size != metadata.get('size'):
+                report['errors'].append(f'{snapshot.pk}: missing or truncated {result.plugin}/{relative}')
+if complete:
+    report['status'] = 'failed' if report['errors'] else 'passed'
+print('STAGING_REPORT=' + json.dumps(report, default=str))
+PY
+)"
+docker exec -e "ACCEPTANCE_SHA=$source_sha" "$container" archivebox shell -c "$script"

bin/verify-staging-release.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Fail closed unless both live test servers accepted this exact release source.
+
+CI tests alone do not establish that the automatically deployed image works on
+our real servers. In particular, a stable PyPI upload must not overtake a stale
+Docker :dev deployment. Only our staging workflow can provide this evidence;
+own-version changes are normalized by the shared fingerprint helper, while
+dependency pins, application code, and CI changes must still match exactly.
+"""
+
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+
+def tested_abx_dl_digest(artifact_dir, expected_sha):
+    artifact_dir = Path(artifact_dir)
+    source_ref = (artifact_dir / "source-ref").read_text().strip()
+    if not re.fullmatch(r"[0-9a-f]{40}", expected_sha) or source_ref != expected_sha:
+        raise SystemExit("Stable publication blocked: tested Docker artifacts belong to a different source commit.")
+
+    image_refs = []
+    for platform in ("linux-amd64", "linux-arm64"):
+        path = artifact_dir / f"abx-dl-image-digest-{platform}"
+        image_ref = path.read_text().strip()
+        match = re.search(r"@(?P<digest>sha256:[0-9a-f]{64})$", image_ref)
+        if not match:
+            raise SystemExit(f"Stable publication blocked: {path.name} is not pinned to a sha256 digest.")
+        image_refs.append(match.group("digest"))
+    if image_refs[0] != image_refs[1]:
+        raise SystemExit("Stable publication blocked: amd64 and arm64 Docker builds used different abx-dl images.")
+    return image_refs[0]
+
+
+def github(path):
+    return json.loads(subprocess.check_output(["gh", "api", path], text=True))
+
+
+def main():
+    repo = os.environ["GITHUB_REPOSITORY"]
+    sha = os.environ["RELEASE_SHA"]
+    fingerprint = subprocess.check_output(
+        [sys.executable, str(Path(__file__).with_name("release-source-fingerprint.py")), sha],
+        text=True,
+    ).strip()
+    expected_abx_dl_digest = tested_abx_dl_digest(os.environ["STAGING_DOCKER_ARTIFACT_DIR"], sha)
+    for environment in ("cabbage", "digestbox"):
+        deployments = github(f"repos/{repo}/deployments?environment={environment}&per_page=100")
+        accepted = False
+        for deployment in deployments:
+            payload = deployment.get("payload") or {}
+            if not isinstance(payload, dict) or payload.get("source_fingerprint") != fingerprint:
+                continue
+            if deployment["creator"]["login"] != "github-actions[bot]" or payload.get("acceptance_version") != 1:
+                continue
+            statuses = github(f"repos/{repo}/deployments/{deployment['id']}/statuses?per_page=1")
+            if not statuses or statuses[0]["state"] != "success":
+                continue
+            deployed_sha = deployment["sha"]
+            if not re.fullmatch(r"[0-9a-f]{40}", deployed_sha):
+                continue
+            inspection = payload.get("inspection") or {}
+            if inspection.get("revision") != deployed_sha or inspection.get("health") != "healthy":
+                continue
+            if not re.fullmatch(r"sha256:[0-9a-f]{64}", inspection.get("image_id", "")):
+                continue
+            abx_dl_image = inspection.get("abx_dl_image", "")
+            abx_dl_match = re.search(r"@(?P<digest>sha256:[0-9a-f]{64})$", abx_dl_image) if isinstance(abx_dl_image, str) else None
+            if not abx_dl_match or abx_dl_match.group("digest") != expected_abx_dl_digest:
+                continue
+            deployed_fingerprint = subprocess.check_output(
+                [sys.executable, str(Path(__file__).with_name("release-source-fingerprint.py")), deployed_sha],
+                text=True,
+            ).strip()
+            if deployed_fingerprint != fingerprint:
+                continue
+            run_id = str(payload.get("run_id", ""))
+            if not run_id.isdecimal():
+                continue
+            run = github(f"repos/{repo}/actions/runs/{run_id}")
+            if (
+                run["path"] != ".github/workflows/staging-acceptance.yml"
+                or run["event"] != "workflow_run"
+                or run["status"] != "completed"
+                or run["conclusion"] != "success"
+            ):
+                continue
+            print(f"{environment}: accepted {deployment['sha']} with abx-dl {expected_abx_dl_digest} ({run['html_url']})")
+            accepted = True
+            break
+        if not accepted:
+            raise SystemExit(f"Stable publication blocked: {environment} has not accepted source fingerprint {fingerprint}.")
+
+
+if __name__ == "__main__":
+    main()

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.65rc5",
+  "version": "0.9.65rc15",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.65rc5"
+version = "0.9.65rc15"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,8 +79,8 @@ dependencies = [
     ### Binary/Package Management
     "abxbus==2.5.65",                 # direct imports only; version constrained by abx-dl -> abx-plugins
     "abxpkg==1.13.11",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins[opencode]==1.13.74", # optional AI runtime clients are owned by the plugin
-    "abx-dl==1.13.51",       # shared ArchiveBox downloader package
+    "abx-plugins[opencode]==1.13.77", # optional AI runtime clients are owned by the plugin
+    "abx-dl==1.13.53",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
     "browser-cookie3>=0.20.1",
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.65rc5"
+current_version = "v0.9.65rc15"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-21T19:45:23.438307361Z"
+exclude-newer = "2026-09-21T22:18:15.120285209Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-26T19:45:22.438320305Z", span = "PT1S" }
+abxbus = { timestamp = "2026-09-26T22:18:14.120299325Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.13.51"
+version = "1.13.53"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/bb/11/fb5c84fb991dba472fc2d3616cc6794df45e0b19e54e0e3016720e4d5014/abx_dl-1.13.51.tar.gz", hash = "sha256:33ba928396ea2f87b23d2cb0c1439e39f64c8b3a9a35451608ae2c2b93c21f65", size = 97366 }
+sdist = { url = "https://files.pythonhosted.org/packages/5c/f1/1c4669e40dc9f6d3e7e943595a2c2b0d795a27ee1460ae47ea430c7386e8/abx_dl-1.13.53.tar.gz", hash = "sha256:48f0e945e35333add056bfab85fcf51c90596ba226cf8a985dcfa57040818334", size = 97356 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d1/d3/a1d92a76e1a7854d5682a062e2339d91021fc5bb7cae6b99137ad42ab0fb/abx_dl-1.13.51-py3-none-any.whl", hash = "sha256:e57c15e31bfa9a79edcf492ec8aa6b57d94d42a7eb5a5021611758d46eab5838", size = 102834 },
+    { url = "https://files.pythonhosted.org/packages/e2/24/787d68c2970ed6ee044e71d0ffbbb68c80983190838e8dc8905bdd1014df/abx_dl-1.13.53-py3-none-any.whl", hash = "sha256:eccd10396a422693e691adf481947a18a522203a8f714644f2597f56d0a6d944", size = 102834 },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.13.74"
+version = "1.13.77"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/89/ce/632a9874d19e24b3209e1a202096d5b501ee4e201dc5a70089b6c25fc004/abx_plugins-1.13.74.tar.gz", hash = "sha256:a22613a4e0a67f7deaba4ccb9010c1d0ed085b0ef490597be9646fed778c272a", size = 452403, upload-time = "2026-09-26T18:48:35.101Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/60/ee/00458913bee25ffc72b86c08355a7dc670c7e4da67ab9a3c9521d13367d8/abx_plugins-1.13.77.tar.gz", hash = "sha256:e1bb3399e076e65348e79889ea48c6e124fb05b4f0e6df5a5f200e29e54fce07", size = 453038, upload-time = "2026-09-26T22:05:36.087Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/10/90/bcf6577f5ed1d8f1b72fc59ab240d5223d08d072fabcd37bac3d7d542fc9/abx_plugins-1.13.74-py3-none-any.whl", hash = "sha256:ea133f3d10f83da129b7c4b964f4c1fbd3e7f7e80d84108f741938c6ab548003", size = 661641, upload-time = "2026-09-26T18:48:33.446Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/99/c7fc8a8437d02d336888ed7b9fe41c04108b0a59022ef8bbc9d0ac8cc7f5/abx_plugins-1.13.77-py3-none-any.whl", hash = "sha256:c9d74aab61e09e79766f79cb8ce9b4844d74af7f74d3a4d37b7ba2fdcb67fe50", size = 662367, upload-time = "2026-09-26T22:05:37.751Z" },
 ]
 
 [package.optional-dependencies]
@@ -127,7 +127,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.65rc5"
+version = "0.9.65rc15"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -227,8 +227,8 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.13.51" },
-    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.74" },
+    { name = "abx-dl", specifier = "==1.13.53" },
+    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.77" },
     { name = "abxbus", specifier = "==2.5.65" },
     { name = "abxpkg", specifier = "==1.13.11" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },