Bump release version to 0.9.72rc15

Resolve relative cookie files before hooks change cwd, reconcile file edits before machine startup saves, and prevent stale cached binaries overriding runtime selections. Cover issue #1896 and real persona localStorage capture through CLI regressions.

.github/dependabot.yml

@@ -15,7 +15,7 @@ updates:
         patterns:
           - "*"
   - package-ecosystem: "npm"
-    directory: "/"
+    directory: "/etc"
     target-branch: "dev"
     schedule:
       interval: "monthly"

archivebox/base_models/models.py

@@ -282,7 +282,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +301,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/rescan.py

@@ -102,9 +102,19 @@ def _reconcile_directory(directory: Path) -> str:
     results = {(item.get("plugin"), item.get("hook_name", "")): item for item in records if item.get("type") == "ArchiveResult"}.values()
     pending = []
     for result in results:
-        result_id = uuid.UUID(result["id"])
         if uuid.UUID(result["snapshot_id"]) != snapshot_id:
             raise ValueError("ArchiveResult belongs to another snapshot")
+        # Legacy JSON conversion produced records without IDs. Recover these
+        # by hook identity while preserving strict collision checks for IDs.
+        if not result.get("id"):
+            if not ArchiveResult.objects.filter(
+                snapshot_id=snapshot_id,
+                plugin=result["plugin"],
+                hook_name=result.get("hook_name", ""),
+            ).exists():
+                pending.append(result)
+            continue
+        result_id = uuid.UUID(result["id"])
         existing = ArchiveResult.objects.filter(pk=result_id).first()
         identity = (snapshot_id, result["plugin"], result.get("hook_name", ""))
         if existing:
@@ -154,7 +164,10 @@ def _reconcile_directory(directory: Path) -> str:
     # Each insert is durable. An interruption after the Snapshot insert is not
     # completion: the next pass still imports missing result IDs and tags.
     for result in pending:
-        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=True)
+        snapshot._create_archive_result_if_missing(result, {}, preserve_identity=bool(result.get("id")))
+    if any(not result.get("id") for result in pending):
+        snapshot.hydrate_archiveresult_output_metadata(snapshot_dir=directory)
+        snapshot.write_index_jsonl(output_dir=directory)
     tags_before = set(snapshot.tags.values_list("name", flat=True))
     snapshot._merge_tags_from_index(record)
     tags_changed = tags_before != set(snapshot.tags.values_list("name", flat=True))

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -468,6 +487,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -825,6 +849,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +875,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_snapshots.py

@@ -52,6 +52,7 @@ from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
 from archivebox.progressmonitor.views import progress_endpoint
+from archivebox.workers.models import RETRY_AT_MAX
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
@@ -131,11 +132,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +993,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1540,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the runner removes and verifies their files."""
+        total = queryset.update(status=Snapshot.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -175,6 +176,19 @@ class SnapshotTag(models.Model):
 class SnapshotQuerySet(models.QuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -541,6 +555,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -574,7 +589,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -661,6 +676,49 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).update(status=self.DELETING_STATE, retry_at=RETRY_AT_MAX)
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        # Finish the bounded read before touching files: even an open SELECT
+        # cursor can hold a SQLite read lock for the duration of slow cleanup.
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""
@@ -1198,6 +1256,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         target_dir = self.get_storage_path_for_version(target)
         if target_dir.exists():
+            self.reconcile_with_index(output_dir=target_dir, update_existing_archive_results=False)
             self.hydrate_archiveresult_output_metadata(snapshot_dir=target_dir)
         if cleanup:
             old_dir, new_dir = cleanup

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/machine/models.py

@@ -255,40 +255,40 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Reconcile once, on the first Machine.current() call in this process.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
         # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine

archivebox/personas/models.py

@@ -74,8 +74,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -156,9 +160,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -315,20 +322,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +354,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/services/runner.py

@@ -65,6 +65,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -738,7 +739,14 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(machine.config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
         if self.persona:
             self.base_config.update(
@@ -1245,6 +1253,7 @@ class CrawlRunner:
             user_config_event = MachineEvent(config=config, config_type="user")
             user_config_event.event_parent_id = crawl_start_event.event_id
             await self.bus.emit(user_config_event).now()
+            derived_config_event = None
             if derived_config:
                 derived_config_event = MachineEvent(config=derived_config, config_type="derived")
                 derived_config_event.event_parent_id = crawl_start_event.event_id
@@ -1260,6 +1269,7 @@ class CrawlRunner:
                 snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
                 abort_requested=self.crawl_is_cancelled,
             )
+            projected = False
             try:
                 snapshot_event = SnapshotEvent(
                     url=snapshot["url"],
@@ -1283,6 +1293,7 @@ class CrawlRunner:
                     raise RuntimeError(f"Snapshot {snapshot_id} did not complete")
                 await completed_snapshot.wait(timeout=snapshot_phase_timeout)
                 await completed_snapshot.event_results_list()
+                projected = True
                 if self.execution_interrupted():
                     return
                 if snapshot["status"] == "sealed":
@@ -1306,6 +1317,16 @@ class CrawlRunner:
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
                 snapshot_service.close()
+                # Projection and snapshot cleanup have finished. Historical
+                # capture results now live in the database; only active work
+                # needs its event ancestry available to the downloader.
+                expired = [snapshot_event, user_config_event] if projected else []
+                if projected and derived_config_event is not None:
+                    expired.append(derived_config_event)
+                if projected:
+                    expired.extend(await self.bus.filter("*", child_of=snapshot_event, past=True, future=False))
+                for event in expired:
+                    event.event_ttl = 0
 
     def seal_snapshot_due_to_limit(self, snapshot_id: str) -> None:
         from archivebox.core.models import Snapshot
@@ -1981,14 +2002,35 @@ def run_pending_crawls(
     last_recovery_at = 0.0
     last_retention_at = 0.0
     last_retention_repair_at = 0.0
+    deletion_thread: threading.Thread | None = None
     last_analyze_at = 0.0
     analyze_queue: list[str] | None = None
     analyze_sweep_started_at = 0.0
     orchestrator_started_at = time.monotonic()
+
+    def delete_in_background() -> None:
+        from django.db import connections
+
+        try:
+            Snapshot.delete_requested(batch_size=100)
+        finally:
+            connections.close_all()
+
     while True:
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if daemon:
+                # At most one batch can be doing filesystem I/O. A stuck mount
+                # must not block capture scheduling or runner shutdown; rows
+                # remain pending until cleanup succeeds, including on restart.
+                if (deletion_thread is None or not deletion_thread.is_alive()) and Snapshot.objects.filter(
+                    status=Snapshot.DELETING_STATE,
+                ).exists():
+                    deletion_thread = threading.Thread(target=delete_in_background, name="snapshot-delete", daemon=True)
+                    deletion_thread.start()
+            else:
+                Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read

archivebox/tests/test_cli_persona.py

@@ -50,12 +50,14 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
     """Import an actual Chromium-created session and archive its private page."""
     import hashlib
     import json
+    import shlex
     import subprocess
 
     from werkzeug.wrappers import Response
     from archivebox.tests.conftest import resolve_abxpkg_chrome_env
 
     browser_env = resolve_abxpkg_chrome_env(tmp_path / "lib")
+    browser_env["ABXPKG_LIB_DIR"] = str(tmp_path / "lib")
     source = tmp_path / "browser"
     httpserver.expect_request("/login").respond_with_data(
         '<html><title>Signed in</title><script>localStorage.setItem("persona-preference", "forest-green")</script>Signed in</html>',
@@ -90,6 +92,7 @@ const puppeteer = require(process.argv[1]);
         const page = await browser.newPage();
         await page.goto(process.argv[4]);
         console.log(await page.title());
+        console.log(await page.evaluate(() => localStorage.getItem('persona-preference')));
     } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exit(1); });
 """,
@@ -104,6 +107,7 @@ const puppeteer = require(process.argv[1]);
     )
     assert seeded.returncode == 0, seeded.stderr
     assert "Signed in" in seeded.stdout
+    assert "forest-green" in seeded.stdout
     preferences = source / "Profile 2" / "Preferences"
     original_preferences = preferences.read_bytes()
 
@@ -161,6 +165,9 @@ const puppeteer = require(process.argv[1]);
         env={**browser_env, "CHROME_SANDBOX": "false"},
     )
     assert archived.returncode == 0, archived.stderr
+    launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
+    assert len(launch_commands) == 1
+    assert shlex.split(launch_commands[0].read_text().splitlines()[1])[0] == browser_env["CHROME_BINARY"]
     outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
     assert len(outputs) == 1
     html = outputs[0].read_text()

archivebox/tests/test_cli_server.py

@@ -448,7 +448,10 @@ def test_port_compatibility_does_not_add_listeners_outside_docker_defaults(tmp_p
         check=True,
     )
     workers = json.loads(result.stdout)
-    assert [worker["name"] for worker in workers] == ["worker_runserver" if debug else "worker_daphne", "worker_runner"]
+    assert [worker["name"] for worker in workers] == [
+        "worker_runserver" if debug else "worker_daphne",
+        "worker_runner",
+    ]
     command = shlex.split(workers[0]["command"])
     assert "--endpoint" not in command
     assert (f"127.0.0.1:{port}" if debug else f"--port={port}") in command

archivebox/tests/test_cli_update.py

@@ -19,6 +19,50 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+def test_rescan_recovers_already_migrated_idless_result(initialized_archive, tmp_path):
+    """Repair collections already migrated by versions affected by #1899."""
+    env = cli_env(disable_extractors=True)
+    run_archivebox_cmd(["add", "--index-only", "https://example.com/legacy-result"], env=env, check=True)
+    run_queued_crawls(initialized_archive, env)
+    with use_archivebox_db(initialized_archive):
+        snapshot = Snapshot.objects.get(url="https://example.com/legacy-result")
+        snapshot.write_index_jsonl()
+        output_dir = snapshot.output_dir
+        snapshot_id = snapshot.pk
+    payload = b"<html>Recovered after a previous migration</html>"
+    (output_dir / "singlefile.html").write_bytes(payload)
+    with (output_dir / "index.jsonl").open("a") as index:
+        index.write(
+            json.dumps(
+                {
+                    "type": "ArchiveResult",
+                    "snapshot_id": str(snapshot_id),
+                    "plugin": "singlefile",
+                    "hook_name": "",
+                    "status": "succeeded",
+                    "output_str": "singlefile.html",
+                    "start_ts": "2021-03-07T23:56:16-08:00",
+                },
+            )
+            + "\n",
+        )
+    result_id = None
+    for expected in ("Repaired: 1", "Unchanged: 1"):
+        result = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert result.returncode == 0, result.stdout + result.stderr
+        assert expected in result.stdout
+        with use_archivebox_db(initialized_archive):
+            archived = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="singlefile")
+            assert archived.status == "succeeded"
+            assert archived.output_size == len(payload)
+            assert archived.output_files
+            assert archived.snapshot.output_size == len(payload)
+            if result_id is not None:
+                assert archived.pk == result_id
+            result_id = archived.pk
+        assert (output_dir / "singlefile.html").read_bytes() == payload
+
+
 def test_rescan_imports_current_layout_and_is_idempotent(initialized_archive, tmp_path):
     env = cli_env(disable_extractors=True)
     run_archivebox_cmd(["add", "--index-only", "https://example.com/rescan"], cwd=initialized_archive, env=env, check=True)

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -15,6 +15,49 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.parametrize("index_format", ["history", "archive_results"])
+def test_rescan_imports_legacy_result_rows_and_saved_content(tmp_path, initialized_archive, index_format):
+    """Issue #1899: orphan metadata must become visible results on the first import."""
+    env = cli_env(disable_extractors=True)
+    legacy_dir = tmp_path / "archive" / "1615190176"
+    legacy_dir.mkdir(parents=True)
+    payload = b"<html><title>Recovered capture</title><body>saved content</body></html>"
+    (legacy_dir / "singlefile.html").write_bytes(payload)
+    entry = {
+        "plugin": "singlefile",
+        "status": "succeeded",
+        "output": "singlefile.html",
+        "start_ts": "2021-03-07T23:56:16-08:00",
+        "end_ts": "2021-03-07T23:56:16-08:00",
+    }
+    metadata = {
+        "url": "https://example.com/recovered",
+        "timestamp": legacy_dir.name,
+        "title": "Recovered capture",
+        index_format: {"singlefile": [entry]} if index_format == "history" else [entry],
+    }
+    original_json = json.dumps(metadata)
+    (legacy_dir / "index.json").write_text(original_json)
+    result_id = None
+    for _ in range(2):
+        command = run_archivebox_cmd(["update", "--rescan", "--migrate-only"], env=env, timeout=60)
+        assert command.returncode == 0, command.stdout + command.stderr
+        with use_archivebox_db(tmp_path):
+            snapshot = Snapshot.objects.get(url=metadata["url"])
+            result = snapshot.archiveresult_set.get(plugin="singlefile")
+            assert result.status == "succeeded"
+            assert result.output_size == len(payload)
+            assert result.output_files
+            assert snapshot.status == "sealed"
+            assert snapshot.archiveresult_set.count() == 1
+            assert snapshot.output_size == len(payload)
+            assert (snapshot.output_dir / "singlefile.html").read_bytes() == payload
+            assert (snapshot.output_dir / "index.json").read_text() == original_json
+            if result_id is not None:
+                assert result.pk == result_id
+            result_id = result.pk
+
+
 def test_update_imports_orphaned_snapshots(tmp_path, initialized_archive):
     """Test that archivebox update imports real legacy archive directories."""
     env = cli_env(disable_extractors=True)

archivebox/tests/test_core_config.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 from pydantic import ValidationError
@@ -95,3 +96,40 @@ def test_resolving_scoped_config_does_not_mutate_process_environment(tmp_path):
             os.environ.pop("ABXPKG_LIB_DIR", None)
         else:
             os.environ["ABXPKG_LIB_DIR"] = previous_lib_dir
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE", "WGET_COOKIES_FILE", "YTDLP_COOKIES_FILE", "GALLERYDL_COOKIES_FILE"])
+def test_cookie_paths_remain_collection_relative_after_runtime_overlays(key, tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: "auth/cookies.txt"})
+    assert str(config[key]) == str(CONSTANTS.DATA_DIR / "auth/cookies.txt")
+    runtime = config.for_crawl_runtime(runtime_overrides={key: "other/cookies.txt"})
+    assert runtime[key] == str(CONSTANTS.DATA_DIR / "other/cookies.txt")
+    absolute = tmp_path / "absolute-cookies.txt"
+    config = ArchiveBoxConfig(**{key: str(absolute)})
+    assert str(config[key]) == str(absolute)
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE"])
+@pytest.mark.parametrize("value", ["", "  ", None])
+def test_empty_cookie_path_disables_import(key, value):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: value})
+    assert config[key] is None
+    assert key not in config.for_crawl_runtime()
+
+
+def test_cookie_runtime_path_preserves_configured_symlink(tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    export = tmp_path / "export.txt"
+    export.write_text("# Netscape HTTP Cookie File\n")
+    configured = tmp_path / "cookies.txt"
+    configured.symlink_to(export)
+    config = ArchiveBoxConfig(COOKIES_FILE=configured)
+    assert str(config.COOKIES_FILE) == str(configured)
+    runtime = config.for_crawl_runtime()
+    assert runtime["COOKIES_FILE"] == str(configured)
+    assert Path(runtime["COOKIES_FILE"]).read_bytes() == export.read_bytes()

archivebox/tests/test_crawl_history.py

@@ -0,0 +1,111 @@
+"""Completed captures belong in the database, not a crawl-long event history."""
+
+import asyncio
+import gc
+import json
+import threading
+import weakref
+from collections import Counter
+
+import pytest
+
+from abx_dl.events import ProcessCompletedEvent, SnapshotCompletedEvent, SnapshotEvent
+from werkzeug import Response
+from archivebox.base_models.models import get_or_create_system_user_pk
+from archivebox.core.models import ArchiveResult, Snapshot
+from archivebox.crawls.models import Crawl
+from archivebox.services.runner import CrawlRunner, run_install
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("timeout_page", [False, True])
+def test_completed_snapshots_expire_after_database_projection(tmp_path, httpserver, hermetic_lib_dir, timeout_page):
+    urls = []
+    response_started = threading.Event()
+    release_response = threading.Event()
+
+    def stalled_response(_request):
+        def body():
+            response_started.set()
+            yield b"<html><head><title>Saved page 3</title></head><body>"
+            release_response.wait()
+            yield b"Completed response after the browser timed out.</body></html>"
+
+        return Response(body(), content_type="text/html")
+
+    httpserver.expect_request("/favicon.ico").respond_with_data("", status=204)
+    for index in range(8):
+        path = f"/page/{index}"
+        if timeout_page and index == 3:
+            httpserver.expect_request(path).respond_with_handler(stalled_response)
+            urls.append(httpserver.url_for(path))
+            continue
+        httpserver.expect_request(path).respond_with_data(
+            f"<html><head><title>Saved page {index}</title></head><body>"
+            "This real page is part of a multi-snapshot crawl whose completed captures "
+            "must remain available in the database after their events expire.</body></html>",
+            content_type="text/html",
+        )
+        urls.append(httpserver.url_for(path))
+    run_install(plugin_names=["title"])
+    crawl = Crawl.objects.create(
+        urls="\n".join(urls),
+        config={
+            "ABXPKG_LIB_DIR": str(hermetic_lib_dir),
+            "PLUGINS": "title",
+            "CRAWL_MAX_CONCURRENT_SNAPSHOTS": 1,
+            "CHROME_PAGELOAD_TIMEOUT": 5,
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
+    runner.bus.event_history.max_history_size = 120
+    sizes = []
+    references = []
+    counts = Counter()
+    retired_ids = set()
+    retained_completed = []
+
+    def count_event(event):
+        counts[event.event_type] += 1
+
+    def observe_snapshot(event: SnapshotEvent):
+        sizes.append(sum(isinstance(previous, SnapshotEvent) for previous in runner.bus.event_history.values()))
+        retained_completed.append(len(retired_ids.intersection(runner.bus.event_history)))
+        references.append(weakref.ref(event))
+
+    async def observe_completed(event: SnapshotCompletedEvent):
+        root = await runner.bus.find(SnapshotEvent, snapshot_id=event.snapshot_id, past=True, future=False)
+        assert root is not None
+        retired_ids.add(root.event_id)
+        retired_ids.update(item.event_id for item in await runner.bus.filter("*", child_of=root, past=True, future=False))
+
+    def release_timed_out_response(event: ProcessCompletedEvent):
+        if response_started.is_set() and event.hook_name == "on_Snapshot__30_chrome_navigate" and event.status == "failed":
+            release_response.set()
+
+    runner.bus.on("*", count_event)
+    runner.bus.on(SnapshotEvent, observe_snapshot)
+    runner.bus.on(SnapshotCompletedEvent, observe_completed)
+    runner.bus.on(ProcessCompletedEvent, release_timed_out_response)
+    try:
+        asyncio.run(runner.run())
+    finally:
+        release_response.set()
+    (tmp_path / "event-counts.json").write_text(json.dumps({"events": counts, "snapshots_in_history": sizes}))
+    snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("url"))
+    assert len(snapshots) == len(urls)
+    assert all(snapshot.status == "sealed" for snapshot in snapshots)
+    for index, snapshot in enumerate(snapshots):
+        if timeout_page and index == 3:
+            assert response_started.is_set()
+            result = ArchiveResult.objects.get(snapshot=snapshot, hook_name="on_Snapshot__30_chrome_navigate", status="failed")
+            assert "timeout" in result.output_str.lower()
+            continue
+        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", status="succeeded")
+        assert result.output_str == f"Saved page {index}"
+        assert (snapshot.output_dir / "title" / "title.txt").read_text().strip() == f"Saved page {index}"
+    assert sizes == [1] * len(urls), sizes
+    assert retained_completed == [0] * len(urls), retained_completed
+    gc.collect()
+    assert all(reference() is None for reference in references)

archivebox/tests/test_issue_1896.py

@@ -0,0 +1,184 @@
+"""Regression coverage for collection cookie paths and config edits on upgrade."""
+
+import json
+import sqlite3
+from contextlib import closing
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+from archivebox.config.configset import read_ini_config
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+@pytest.mark.parametrize("persona_name", ["Default", "default"])
+def test_add_preserves_case_distinct_personas(initialized_archive, httpserver, persona_name):
+    """Both names from the follow-up report remain usable without deleting either."""
+    run_archivebox_cmd(["persona", "create", "Default", "default"], cwd=initialized_archive, check=True)
+    persona_dir = initialized_archive / "personas" / persona_name
+    (persona_dir / "auth.json").touch()
+    (persona_dir / "cookies.txt").touch()
+    httpserver.expect_request("/").respond_with_data(
+        "<html><head><title>Existing persona capture</title></head><body>"
+        "Both personas retained. This saved page verifies that choosing either existing "
+        "persona still captures the full document without deleting browser state.</body></html>",
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--persona", persona_name, "--plugins=dom", httpserver.url_for("/")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert "Both personas retained" in outputs[0].read_text()
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT name FROM personas_persona ORDER BY name").fetchall() == [("Default",), ("default",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_envs = db.execute("SELECT env FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_envs
+        assert all(json.loads(row[0])["ACTIVE_PERSONA"] == persona_name for row in hook_envs)
+
+
+@pytest.mark.parametrize("contents,explicit", [("", True), ("not JSON", True), ("not JSON", False)])
+def test_add_rejects_invalid_persona_auth_with_file_path(initialized_archive, httpserver, contents, explicit):
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.write_text(contents)
+    if explicit:
+        run_archivebox_cmd(
+            ["config", "--set", "AUTH_STORAGE_FILE=personas/Default/auth.json"],
+            cwd=initialized_archive,
+            check=True,
+        )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 1, result.stdout + result.stderr
+    logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    assert logs
+    assert f"Invalid JSON cookie export: {auth_file}" in "\n".join(log.read_text() for log in logs)
+    assert not list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert auth_file.read_text() == contents
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("isolation", ["crawl", "snapshot"])
+def test_add_with_empty_autodiscovered_persona_auth(initialized_archive, httpserver, cookie_file_exists, isolation):
+    """An empty auth placeholder must not block a crawl or hide Netscape cookies."""
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.touch()
+    cookies_file = persona_dir / "cookies.txt"
+    cookies_file.write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n" if cookie_file_exists else "",
+    )
+    httpserver.expect_request("/").respond_with_data(
+        '<html><title>Empty persona auth</title><body><output id="cookies"></output>'
+        '<script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false", "CHROME_ISOLATION": isolation},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert ("issue1896=imported" in outputs[0].read_text()) == cookie_file_exists
+    assert auth_file.read_bytes() == b""
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT cmd, env, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for cmd, env, exit_code in hook_rows:
+            hook_env = json.loads(env)
+            assert not hook_env.get("AUTH_STORAGE_FILE")
+            expected_cookies = cookies_file if isolation == "crawl" else Path(hook_env["PERSONAS_DIR"]) / "Default" / "cookies.txt"
+            assert hook_env["COOKIES_FILE"] == str(expected_cookies)
+            hook_name = Path(json.loads(cmd)[0]).name
+            # The crawl wait hook explicitly skips when each snapshot owns Chrome.
+            assert exit_code == (10 if isolation == "snapshot" and hook_name == "on_CrawlSetup__91_chrome_wait.js" else 0)
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+def test_add_with_collection_relative_cookies(initialized_archive, httpserver, cookie_file_exists):
+    """Real crawl hooks must read cookies from the collection, not their cwd."""
+    if cookie_file_exists:
+        (initialized_archive / "cookies.txt").write_text(
+            "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+        )
+    httpserver.expect_request("/").respond_with_data(
+        "<html><title>Issue 1896</title><body><p>Collection cookie import regression: "
+        "this page must still be archived when the optional cookie file is absent.</p>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    run_archivebox_cmd(
+        ["config", "--set", "COOKIES_FILE=cookies.txt"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    html = outputs[0].read_text()
+    assert ("issue1896=imported" in html) == cookie_file_exists
+    launch_logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    logs = "\n".join(log.read_text() for log in launch_logs)
+    if cookie_file_exists:
+        assert f"from {initialized_archive / 'cookies.txt'}" in logs
+    else:
+        assert "Cookies file not found" in logs
+        assert "continuing without cookie import" in logs
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT env, status, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for env, status, exit_code in hook_rows:
+            assert json.loads(env)["COOKIES_FILE"] == str(initialized_archive / "cookies.txt")
+            assert (status, exit_code) == ("exited", 0)
+
+
+@pytest.mark.parametrize("enabled_key", ["USE_CHROME", "CHROME_ENABLED"])
+@pytest.mark.parametrize("startup_save", ["binary_sanitation", "host_refresh"])
+def test_init_preserves_edited_use_chrome_before_startup_save(initialized_archive, enabled_key, startup_save):
+    """Startup saves must not overwrite a newer file before config sync."""
+    config_path = initialized_archive / "ArchiveBox.conf"
+    config_path.write_text(config_path.read_text() + f"\n[SERVER_CONFIG]\n{enabled_key} = True\n")
+    run_archivebox_cmd(
+        ["config", "--set", f"WGET_BINARY={initialized_archive / 'removed-wget'}"]
+        if startup_save == "binary_sanitation"
+        else ["config", "--set", "TIMEOUT=37"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    if startup_save == "host_refresh":
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            db.execute("UPDATE machine_machine SET modified_at=?", [(datetime.now(timezone.utc) - timedelta(days=8)).isoformat()])
+            db.commit()
+    config_path.write_text(config_path.read_text().replace(f"{enabled_key} = True", f"{enabled_key} = False"))
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    run_archivebox_cmd(["init"], cwd=initialized_archive, timeout=120, check=True)
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    result = run_archivebox_cmd(["config", "--get", "CHROME_ENABLED"], cwd=initialized_archive)
+    assert result.returncode == 0, result.stderr
+    assert "CHROME_ENABLED = false" in result.stdout

archivebox/tests/test_runner_deletion.py

@@ -0,0 +1,177 @@
+"""Exercise queued deletion in the existing runner with a real server."""
+
+import json
+import textwrap
+import time
+from pathlib import Path
+
+import pytest
+
+from archivebox.tests.conftest import (
+    cli_env,
+    get_free_port,
+    kill_processes_for_data_dir,
+    run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    wait_for_log,
+)
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+
+@pytest.mark.django_db(transaction=True)
+@pytest.mark.parametrize("restart", [False, True])
+def test_runner_retries_failed_deletion_without_blocking_captures(initialized_archive, recursive_test_site, restart):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+
+    port = get_free_port()
+    env = cli_env(
+        live=True,
+        server=True,
+        port=port,
+        PLUGINS="title",
+        SAVE_TITLE="True",
+        SEARCH_BACKEND_ENGINE="ripgrep",
+    )
+    prepared = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            "import json; from archivebox.crawls.models import Crawl; "
+            "from archivebox.core.models import Snapshot, ArchiveResult; "
+            "from archivebox.base_models.models import get_or_create_system_user_pk; "
+            "c=Crawl.objects.create(urls='https://example.com/delete',created_by_id=get_or_create_system_user_pk(),status='sealed'); "
+            "s=Snapshot.objects.create(url='https://example.com/delete',crawl=c,status='deleting'); "
+            "p=s.output_dir/'wget'; p.mkdir(parents=True,exist_ok=True); "
+            "(p/'index.html').write_text('keep this until cleanup succeeds'); "
+            "ArchiveResult.objects.create(snapshot=s,plugin='wget',hook_name='on_Snapshot__06_wget',status='succeeded'); "
+            "p.chmod(0o500); print(json.dumps({'id':str(s.pk),'path':str(p)}))",
+        ],
+        cwd=initialized_archive,
+        env=env,
+    )
+    assert prepared.returncode == 0, prepared.stderr or prepared.stdout
+    fixture = json.loads(prepared.stdout.strip().splitlines()[-1])
+    protected = Path(fixture["path"])
+    worker_log = initialized_archive / "logs" / "worker_runner.log"
+    server = None
+    try:
+        server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-worker-server.log")
+        wait_for_log(worker_log, "PermissionError", timeout=30)
+        with use_archivebox_db(initialized_archive):
+            assert Process.objects.filter(worker_type="worker_runner", status="running").exists()
+            assert not Process.objects.filter(worker_type="worker_snapshot_delete").exists()
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+            assert ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert (protected / "index.html").is_file()
+        added = run_archivebox_cmd(
+            ["add", "--bg", "--plugins=title", recursive_test_site["root_url"]],
+            cwd=initialized_archive,
+            env=env,
+        )
+        assert added.returncode == 0, added.stderr or added.stdout
+        deadline = time.monotonic() + 60
+        with use_archivebox_db(initialized_archive):
+            while time.monotonic() < deadline:
+                captured = Snapshot.objects.filter(url=recursive_test_site["root_url"], status="sealed").first()
+                if captured is not None:
+                    break
+                assert server.poll() is None
+                time.sleep(0.1)
+            else:
+                pytest.fail((initialized_archive / "logs" / "worker_runner.log").read_text()[-12000:])
+            result = ArchiveResult.objects.get(snapshot=captured, plugin="title", status="succeeded")
+            assert result.output_files
+            assert captured.title == "Root"
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+        if restart:
+            stop_archivebox_process(server)
+            server = None
+        protected.chmod(0o700)
+        if restart:
+            server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-restarted-server.log")
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(initialized_archive):
+            while Snapshot.objects.filter(pk=fixture["id"]).exists() and time.monotonic() < deadline:
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=fixture["id"]).exists(), worker_log.read_text()
+            assert not ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert not protected.parent.exists()
+    finally:
+        if protected.exists():
+            protected.chmod(0o700)
+        if server is not None:
+            stop_archivebox_process(server)
+        kill_processes_for_data_dir(initialized_archive)
+
+
+def test_deletion_releases_database_before_filesystem_io(initialized_archive):
+    result = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            textwrap.dedent("""
+                import json
+                import sqlite3
+                import sys
+                from pathlib import Path
+                from django.db import connection
+                from archivebox.base_models.models import get_or_create_system_user_pk
+                from archivebox.core.models import ArchiveResult, Snapshot
+                from archivebox.crawls.models import Crawl
+                from archivebox.services.runner import run_pending_crawls
+
+                # DELETE mode exposes both read-cursor and transaction locks.
+                with connection.cursor() as cursor:
+                    cursor.execute('PRAGMA journal_mode=DELETE')
+                    assert cursor.fetchone()[0] == 'delete'
+                crawl = Crawl.objects.create(
+                    urls='https://example.com/delete',
+                    created_by_id=get_or_create_system_user_pk(), status='sealed',
+                )
+                paths = set()
+                snapshot_ids = []
+                for i in range(2):
+                    snapshot = Snapshot.objects.create(url=f'https://example.com/delete/{i}', crawl=crawl, status='deleting')
+                    path = snapshot.output_dir
+                    path.mkdir(parents=True, exist_ok=True)
+                    (path / 'saved.html').write_text('real saved output')
+                    ArchiveResult.objects.create(snapshot=snapshot, plugin='wget', status='succeeded')
+                    paths.add(str(path))
+                    snapshot_ids.append(snapshot.pk)
+                checked = set()
+
+                def inspect_filesystem_operation(event, args):
+                    if event != 'shutil.rmtree' or str(args[0]) not in paths:
+                        return
+                    assert not connection.in_atomic_block
+                    # Observe the real rmtree call without replacing it. An
+                    # independent writer must acquire its lock immediately.
+                    writer = sqlite3.connect(connection.settings_dict['NAME'], timeout=0, isolation_level=None)
+                    try:
+                        writer.execute('BEGIN EXCLUSIVE')
+                        writer.execute('UPDATE crawls_crawl SET notes=? WHERE id=?', ('concurrent write succeeded', crawl.pk.hex))
+                        writer.execute('COMMIT')
+                    finally:
+                        writer.close()
+                    checked.add(str(args[0]))
+
+                sys.addaudithook(inspect_filesystem_operation)
+                run_pending_crawls(maintenance_only=True)
+                assert checked == paths
+                assert not Snapshot.objects.filter(crawl=crawl).exists()
+                assert not ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids).exists()
+                assert all(not Path(path).exists() for path in paths)
+                crawl.refresh_from_db()
+                assert crawl.notes == 'concurrent write succeeded'
+                print(json.dumps({'verified_deletions': len(checked)}))
+            """),
+        ],
+        cwd=initialized_archive,
+        env=cli_env(disable_extractors=True),
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    assert json.loads(result.stdout.strip().splitlines()[-1]) == {"verified_deletions": 2}

archivebox/tests/test_ui_admin_crawl.py

@@ -343,6 +343,7 @@ def test_discovered_snapshots_inherit_current_crawl_tags(crawl):
     assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl", "discovered"}
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, admin_user):
     crawl = Crawl.objects.create(
         urls="https://example.com/remove-me",
@@ -352,6 +353,9 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
         crawl=crawl,
         url="https://example.com/remove-me",
     )
+    output_dir = snapshot.output_dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "captured.txt").write_text("remove this capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -362,11 +366,13 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
     assert response.status_code == 200
     assert response.json()["ok"] is True
     assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
+    assert not output_dir.exists()
 
     crawl.refresh_from_db()
     assert "https://example.com/remove-me" not in crawl.urls
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(client, admin_user):
     crawl = Crawl.objects.create(
         urls=("https://cdn.example.com/asset.js\nhttps://cdn.example.com/second.js\nhttps://example.com/root"),
@@ -382,6 +388,12 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
         url="https://example.com/root",
         status=Snapshot.StatusChoices.SEALED,
     )
+    removed_dir = queued_snapshot.output_dir
+    removed_dir.mkdir(parents=True, exist_ok=True)
+    (removed_dir / "captured.txt").write_text("excluded capture")
+    preserved_file = preserved_snapshot.output_dir / "captured.txt"
+    preserved_file.parent.mkdir(parents=True, exist_ok=True)
+    preserved_file.write_text("preserved capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -401,6 +413,8 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
     assert "https://example.com/root" in crawl.urls
     assert not Snapshot.objects.filter(pk=queued_snapshot.pk).exists()
     assert Snapshot.objects.filter(pk=preserved_snapshot.pk).exists()
+    assert not removed_dir.exists()
+    assert preserved_file.read_text() == "preserved capture"
 
 
 def test_snapshot_from_json_trims_markdown_suffixes_on_discovered_urls(crawl):

archivebox/tests/test_ui_admin_snapshot.py

@@ -26,6 +26,65 @@ pytestmark = pytest.mark.django_db(transaction=True)
 REPO_ROOT = Path(__file__).resolve().parents[2]
 
 
+def test_bulk_snapshot_delete_keeps_rows_until_worker_verifies_files(client, admin_user, crawl):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.runner import run_pending_crawls
+
+    snapshots = [Snapshot.objects.create(url=f"https://example.com/delete/{i}", crawl=crawl) for i in range(2)]
+    output_dirs = [snapshot.output_dir for snapshot in snapshots]
+    for snapshot, output_dir in zip(snapshots, output_dirs):
+        (output_dir / "wget").mkdir(parents=True, exist_ok=True)
+        (output_dir / "wget" / "index.html").write_text("real saved output")
+        ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    ids = [snapshot.pk for snapshot in snapshots]
+    client.force_login(admin_user)
+    response = client.post(
+        reverse("admin:core_snapshot_changelist"),
+        {"action": "delete_snapshots", ACTION_CHECKBOX_NAME: [str(pk) for pk in ids]},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302
+    assert Snapshot.objects.filter(pk__in=ids, status="deleting").count() == 2
+    assert ArchiveResult.objects.filter(snapshot_id__in=ids).count() == 2
+    assert all((path / "wget" / "index.html").is_file() for path in output_dirs)
+    pending = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST)
+    assert pending.status_code == 200
+    assert "Deleting" in pending.content.decode()
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk__in=ids).exists()
+    assert not ArchiveResult.objects.filter(snapshot_id__in=ids).exists()
+    assert not any(path.exists() for path in output_dirs)
+
+
+def test_snapshot_delete_preserves_rows_when_real_filesystem_cleanup_fails(snapshot):
+    from archivebox.core.models import ArchiveResult, Snapshot
+
+    output_dir = snapshot.output_dir
+    protected = output_dir / "wget"
+    protected.mkdir(parents=True, exist_ok=True)
+    saved_file = protected / "index.html"
+    saved_file.write_text("must not lose the row before this file is removed")
+    result = ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    snapshot_id = snapshot.pk
+    protected.chmod(0o500)
+    try:
+        with pytest.raises(PermissionError):
+            Snapshot.objects.filter(pk=snapshot_id).delete()
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert ArchiveResult.objects.filter(pk=result.pk).exists()
+        assert saved_file.is_file()
+        from archivebox.services.runner import run_pending_crawls
+
+        run_pending_crawls(maintenance_only=True)
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert saved_file.is_file()
+    finally:
+        protected.chmod(0o700)
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+    assert not output_dir.exists()
+
+
 def test_failed_hook_partial_file_does_not_become_successful_card(snapshot, cached_abxpkg_lib_dir):
     partial = Path(snapshot.output_dir) / "title" / "partial.html"
     partial.parent.mkdir(parents=True, exist_ok=True)

archivebox/tests/test_ui_config_views.py

@@ -7,7 +7,7 @@ from archivebox.plugins import views as plugin_views
 from archivebox.core import views as core_views
 from archivebox.config import CONSTANTS
 from archivebox.machine.models import Machine
-from archivebox.tests.conftest import install_real_binary
+from archivebox.tests.conftest import ADMIN_TEST_HOST, install_real_binary
 
 
 pytestmark = pytest.mark.django_db
@@ -75,18 +75,42 @@ def test_binaries_list_view_uses_db_version_and_hides_youtube_dl_alias(admin_req
 
     assert len(context["table"]["Binary Name"]) == 1
     assert str(context["table"]["Binary Name"][0].link_item) == "yt-dlp"
-    assert context["table"]["Found Version"][0] == f"✅ {binary.version}"
+    assert context["table"]["Recorded Version"][0] == binary.version
     assert context["table"]["Provided By"][0] == binary.binprovider
-    assert context["table"]["Found Abspath"][0] == binary.abspath
+    assert context["table"]["Recorded Abspath"][0] == binary.abspath
 
 
 def test_binaries_list_view_only_shows_persisted_records(admin_request):
     context = config_views.binaries_list_view.__wrapped__(admin_request("/admin/environment/binaries/"))
 
     assert context["table"]["Binary Name"] == []
-    assert context["table"]["Found Version"] == []
+    assert context["table"]["Recorded Version"] == []
     assert context["table"]["Provided By"] == []
-    assert context["table"]["Found Abspath"] == []
+    assert context["table"]["Recorded Abspath"] == []
+
+
+@pytest.mark.django_db(transaction=True)
+def test_binaries_page_labels_removed_installation_as_recorded_metadata(admin_client, machine, hermetic_lib_dir):
+    from pathlib import Path
+
+    binary = install_real_binary("node", machine=machine)
+    binary_path = Path(binary.abspath)
+    assert binary_path.is_relative_to(hermetic_lib_dir)
+    assert binary_path.is_symlink()
+    binary_path.unlink()
+    assert not binary_path.exists()
+
+    response = admin_client.get("/admin/environment/binaries/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    html = response.content.decode()
+    assert "/admin/environment/binaries/node/" in html
+    assert "Recorded Version" in html
+    assert "Recorded Abspath" in html
+    assert binary.version in html
+    assert str(binary_path) in html
+    assert f"✅ {binary.version}" not in html
+    binary.refresh_from_db()
+    assert binary.abspath == str(binary_path)
 
 
 @pytest.mark.django_db(transaction=True)

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.71",
+  "version": "0.9.72rc15",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.71"
+version = "0.9.72rc15"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -77,10 +77,10 @@ dependencies = [
     "w3lib>=2.2.1",          # used for parsing content-type encoding from http response headers & html tags
     ### Extractor dependencies (runtime binaries resolved through abxpkg)
     ### Binary/Package Management
-    "abxbus==2.5.70",                 # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abxpkg==1.13.21",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins[opencode]==1.13.109", # optional AI runtime clients are owned by the plugin
-    "abx-dl==1.13.79",       # shared ArchiveBox downloader package
+    "abxbus==2.5.73",                 # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abxpkg==1.13.22",                # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abx-plugins[opencode]==1.13.118", # optional AI runtime clients are owned by the plugin
+    "abx-dl==1.13.84",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
     "browser-cookie3>=0.20.1",
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.71"
+current_version = "v0.9.72rc15"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-23T19:26:02.6709672Z"
+exclude-newer = "2026-09-27T19:34:06.371509767Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-28T19:26:01.670984315Z", span = "PT1S" }
+abxbus = { timestamp = "2026-10-02T19:34:05.371524359Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.13.79"
+version = "1.13.84"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f5/59/e178d330d244351f01026a2c261c7c0d8067c0efcbef9cf095635fa97579/abx_dl-1.13.79.tar.gz", hash = "sha256:8825b06570df48f3ecdd525e8fca09a2d8dac77b8ba059efaac7d0df845b578a", size = 97588, upload-time = "2026-09-28T19:23:47.984Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/e7/27c5a4e4b430b99f6625cad90dba7e7e42e72df2ad980c22935b2faa75f9/abx_dl-1.13.84.tar.gz", hash = "sha256:c936c8dba56cdf5303c4fef81c175191deec812be756ab0378c6158885cc1c75", size = 97709, upload-time = "2026-10-02T19:32:11.007Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/60/6f/f7de1fd42bbcdf56769a7277ca261007525d2a482544fe03d509b3cc6805/abx_dl-1.13.79-py3-none-any.whl", hash = "sha256:3b9b8426da6c0fcd55583d0f26c037561b40982866c4a12f481c8f481fbe56b2", size = 102934, upload-time = "2026-09-28T19:23:49.287Z" },
+    { url = "https://files.pythonhosted.org/packages/02/45/59fdcd0b921cf77e9f32ead85dcd564c822f249601c5edfbb6be244bb0ea/abx_dl-1.13.84-py3-none-any.whl", hash = "sha256:bcad7a0e7dad1a22dee382584f20df1e2359c28e30f00b9464038b09ae26deca", size = 103087, upload-time = "2026-10-02T19:32:09.405Z" },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.13.109"
+version = "1.13.118"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/22/e236ab70ff3c24c3be7f511c202e9943b4acf7988ca180f6593a959cf190/abx_plugins-1.13.109.tar.gz", hash = "sha256:056556249fbe5e1aa5eca3cca84923fab9a4a1fcfd0d3f82c4911de97f2e49f5", size = 461059, upload-time = "2026-09-28T19:08:05.06Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/44/c5/91219063cb0931aefb8d93b18cc367930f030c62f56740bad1dea3d11e24/abx_plugins-1.13.118.tar.gz", hash = "sha256:bcc0db30497fd39c7d550871e7c25c8cded41517d24681326897279c58fd862d", size = 461690, upload-time = "2026-10-02T19:22:11.887Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a4/c9/6331394242f8c5afa1fb03abfd805e509add8a7d03c14b9e126e6724af20/abx_plugins-1.13.109-py3-none-any.whl", hash = "sha256:5bdecde68a9bc2265a26b95205c9214a07e39328c981c8700234f577a57d0ba7", size = 673893, upload-time = "2026-09-28T19:08:06.808Z" },
+    { url = "https://files.pythonhosted.org/packages/2a/42/1edb6ebf174d8b81e9f59dd53188611cbc707e5fad49ab6250649a2ef9a5/abx_plugins-1.13.118-py3-none-any.whl", hash = "sha256:9c1964bdaedcae4071273288fbb070f3397ae83cdd4e6e7091f6f5d909229fdc", size = 674589, upload-time = "2026-10-02T19:22:10.34Z" },
 ]
 
 [package.optional-dependencies]
@@ -68,21 +68,21 @@ opencode = [
 
 [[package]]
 name = "abxbus"
-version = "2.5.70"
+version = "2.5.73"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/52/80/43ab38e5a88a6c8864c2460e0685f946f9fcd6cb4776033611107d70ecd6/abxbus-2.5.70.tar.gz", hash = "sha256:e2a6593d67ed619833a6a2b2a282c58312404f166436a0991f5017d51f45fbe5", size = 134824, upload-time = "2026-09-28T18:36:05.082Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/90/d3/175f38969118aba03326140fdb3d0582ced08026c45cfe4773946d391d0f/abxbus-2.5.73.tar.gz", hash = "sha256:21ed000fdf6dc0e633bac4250d4354e671d05312519aad5078c5bc9e2581dc6a", size = 139209, upload-time = "2026-10-02T18:32:33.762Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/4d/0b/67ebd55f417b641f05b9ba07d738b75b33924026cda220c9d4f771d6e16e/abxbus-2.5.70-py3-none-any.whl", hash = "sha256:a33727a432ec6729d3bd0674d36bb82d87a1faac66480a3eeddaee23e6960a25", size = 128460, upload-time = "2026-09-28T18:36:06.186Z" },
+    { url = "https://files.pythonhosted.org/packages/14/f6/256602ce4b213a13eca3ae2a61659e64881c3f1c9bddd9a4b6d8fb95e37f/abxbus-2.5.73-py3-none-any.whl", hash = "sha256:c20d0bb4dce4ba34ea73ecdae09f118a3a3e72065c7822614f8e7506e0cfb179", size = 132470, upload-time = "2026-10-02T18:32:32.384Z" },
 ]
 
 [[package]]
 name = "abxpkg"
-version = "1.13.21"
+version = "1.13.22"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -90,9 +90,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/0f/bb/56f863aa00e8e9311c0c4af059e7f0ebed2242fd40be9f3b4bbb0e70d646/abxpkg-1.13.21.tar.gz", hash = "sha256:9d5e05a0534836ea637e1c94f4555558c5d6a09368e3ee24e5a056812afc2342", size = 243339, upload-time = "2026-09-28T18:51:45.417Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/30/0c/1f54a00491eeb26b343794951d46cdeab77849c854224d343d1941912f71/abxpkg-1.13.22.tar.gz", hash = "sha256:6bde8929e1df014aa6abfc9256581d0e4587c6d717f5a101d710d03af894f00a", size = 243342, upload-time = "2026-10-02T18:49:33.282Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/37/9c/e60dc8ef5ec6dbe9b39c6bd06165f9d018660cba891295ff625d7cf577b1/abxpkg-1.13.21-py3-none-any.whl", hash = "sha256:b796b99dbbdfdafa3ab05f881ad505539deabbe14eb7b358811f12c47c5a47ad", size = 254735, upload-time = "2026-09-28T18:51:43.415Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/2a/84250b8eeec73530da8afaadcfbc8da87fa340a63d4f7aa18f5553610645/abxpkg-1.13.22-py3-none-any.whl", hash = "sha256:b04b6b79a17d1d876b88fe91ad9c64933f2cc5adf7db44d54ee417f2d71b67a2", size = 254735, upload-time = "2026-10-02T18:49:31.126Z" },
 ]
 
 [[package]]
@@ -127,7 +127,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.71"
+version = "0.9.72rc15"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -227,10 +227,10 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.13.79" },
-    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.109" },
-    { name = "abxbus", specifier = "==2.5.70" },
-    { name = "abxpkg", specifier = "==1.13.21" },
+    { name = "abx-dl", specifier = "==1.13.84" },
+    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.118" },
+    { name = "abxbus", specifier = "==2.5.73" },
+    { name = "abxpkg", specifier = "==1.13.22" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
     { name = "atomicwrites", specifier = "==1.4.1" },
     { name = "base32-crockford", specifier = ">=0.3.0" },