Bump release version to 0.9.72rc11

Resolve relative cookie files before hooks change cwd, reconcile file edits before machine startup saves, and prevent stale cached binaries overriding runtime selections. Cover issue #1896 and real persona localStorage capture through CLI regressions.

archivebox/base_models/models.py

@@ -282,7 +282,14 @@ class ModelWithOutputDir(ModelWithUUID):
             if path.is_symlink() or path.is_file():
                 path.unlink(missing_ok=True)
             elif path.is_dir():
-                shutil.rmtree(path, ignore_errors=True)
+                shutil.rmtree(path)
+            # lstat also catches dangling symlinks and propagates mount and
+            # permission errors rather than mistaking them for absence.
+            try:
+                path.lstat()
+            except FileNotFoundError:
+                continue
+            raise OSError(f"Output path still exists after deletion: {path}")
 
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Capture output paths before DB deletion and remove them after commit."""
@@ -294,7 +301,9 @@ class ModelWithOutputDir(ModelWithUUID):
         if cls._delete_signal_registered:
             return
 
-        def schedule_output_dir_cleanup(sender, instance, using, **kwargs):
+        def schedule_output_dir_cleanup(sender, instance, using, origin=None, **kwargs):
+            if getattr(origin, "_output_files_deleted", False):
+                return
             if not isinstance(instance, ModelWithOutputDir):
                 return
             instance.schedule_delete_cleanup(using=using)

archivebox/cli/archivebox_remove.py

@@ -87,9 +87,8 @@ def remove(
     # Delete one snapshot at a time. Each ``.delete()`` is its own short
     # Django-atomic block, so the writer lock is released between rows and
     # an in-flight daemon transaction can interleave instead of deadlocking.
-    # Filesystem cleanup for each row is scheduled via ``transaction.on_commit``
-    # in ``base_models/models.py`` and runs AFTER its row's tx commits — so
-    # rmtree doesn't hold the lock either.
+    # Snapshot.delete() removes and verifies filesystem outputs first, outside
+    # the short database delete transaction. Failed cleanup retains the row.
     #
     deleted_snapshot_pks = []
     timed_out = False

archivebox/cli/archivebox_run.py

@@ -297,6 +297,10 @@ def run_runner(
 
     shutdown_state = None
     try:
+        if daemon:
+            from archivebox.workers.supervisord_util import SNAPSHOT_DELETE_WORKER, get_or_create_supervisord_process, start_worker
+
+            start_worker(get_or_create_supervisord_process(), SNAPSHOT_DELETE_WORKER())
         with (
             foreground_shutdown_signals(
                 on_signal=_exit_daemon_runner_on_signal if daemon else None,
@@ -328,6 +332,11 @@ def run_runner(
         rprint(f"[red]Runner error: {type(e).__name__}: {e}[/red]", file=sys.stderr)
         return 1
     finally:
+        if daemon:
+            from archivebox.workers.supervisord_util import stop_own_supervisord_process
+
+            if stop_own_supervisord_process():
+                Process.cleanup_stale_running(machine=current.machine)
         current.refresh_from_db()
         if current.status != Process.StatusChoices.EXITED:
             current.mark_exited()

archivebox/config/common.py

@@ -34,6 +34,25 @@ ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
 LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
+
+def _is_cookie_file_config_key(key: str) -> bool:
+    return key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"} or key.endswith("_COOKIES_FILE")
+
+
+def _resolve_cookie_file_paths(config: dict[str, Any]) -> dict[str, Any]:
+    """Keep collection-relative auth paths valid after hooks change cwd."""
+    for key, value in config.items():
+        if _is_cookie_file_config_key(key):
+            if value is None or (isinstance(value, str) and not value.strip()):
+                config[key] = ""
+                continue
+            file_path = Path(value).expanduser()
+            if not file_path.is_absolute():
+                file_path = CONSTANTS.DATA_DIR / file_path
+            config[key] = str(file_path.absolute())
+    return config
+
+
 ###################### Config ##########################
 
 _STDOUT_CONSOLE = Console()
@@ -468,6 +487,11 @@ class ArchivingConfig(BaseConfigSet):
     COOKIES_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     AUTH_STORAGE_FILE: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
+    @field_validator("COOKIES_FILE", "AUTH_STORAGE_FILE", mode="before")
+    @classmethod
+    def empty_cookie_file_is_unset(cls, value):
+        return None if isinstance(value, str) and not value.strip() else value
+
     URL_DENYLIST: str = Field(
         default=(
             r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$"
@@ -825,6 +849,8 @@ class ArchiveBoxBaseConfig(
         if runtime_overrides:
             config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
 
+        _resolve_cookie_file_paths(config)
+
         # Hooks should only see concrete plugin-local flags, never the
         # ArchiveBox selectors used to derive them.
         config.pop("PLUGINS", None)
@@ -849,6 +875,13 @@ class ArchiveBoxBaseConfig(
             lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.ABXPKG_LIB_DIR = lib_dir.resolve()
 
+        cookie_paths = {key: getattr(self, key) for key in type(self).model_fields if _is_cookie_file_config_key(key)}
+        for key, value in _resolve_cookie_file_paths(cookie_paths).items():
+            if key in {"COOKIES_FILE", "AUTH_STORAGE_FILE"}:
+                setattr(self, key, Path(value) if value else None)
+            else:
+                setattr(self, key, value)
+
         return self
 
     @model_validator(mode="after")

archivebox/config/views.py

@@ -150,9 +150,9 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     rows = {
         "Binary Name": [],
-        "Found Version": [],
+        "Recorded Version": [],
         "Provided By": [],
-        "Found Abspath": [],
+        "Recorded Abspath": [],
     }
 
     db_binaries = get_db_binaries_by_name()
@@ -160,18 +160,10 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and binary.is_valid)
-
         rows["Binary Name"].append(ItemLink(name, key=name))
-
-        if binary_is_valid:
-            rows["Found Version"].append(f"✅ {binary.version}" if binary.version else "✅ found")
-            rows["Provided By"].append(binary.binprovider or "-")
-            rows["Found Abspath"].append(binary.abspath or "-")
-        else:
-            rows["Found Version"].append("❌ missing")
-            rows["Provided By"].append("-")
-            rows["Found Abspath"].append("-")
+        rows["Recorded Version"].append(binary.version or "-")
+        rows["Provided By"].append(binary.binprovider or "-")
+        rows["Recorded Abspath"].append(binary.abspath or "-")
 
     return TableContext(
         title="Binaries",

archivebox/core/admin_snapshots.py

@@ -52,6 +52,7 @@ from archivebox.core.models import Tag, Snapshot, ArchiveResult
 from archivebox.crawls.models import Crawl
 from archivebox.core.admin_archiveresults import render_archiveresults_list
 from archivebox.progressmonitor.views import progress_endpoint
+from archivebox.workers.models import RETRY_AT_MAX
 from archivebox.core.permissions import (
     PERMISSIONS_CHOICES,
     PERMISSIONS_META,
@@ -131,11 +132,11 @@ class SnapshotStatusListFilter(admin.SimpleListFilter):
     parameter_name = "snapshot_status"
 
     def lookups(self, request, model_admin):
-        return Snapshot.StatusChoices.choices
+        return Snapshot._meta.get_field("status").choices
 
     def queryset(self, request, queryset):
         value = self.value()
-        if value in Snapshot.StatusChoices.values:
+        if value in dict(Snapshot._meta.get_field("status").choices):
             return queryset.filter(status=value)
         return queryset
 
@@ -992,10 +993,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return format_html(
             '<a class="snapshot-title-detail-hitbox" href="{}" aria-label="Open snapshot details"></a>'
             "{}"
+            "{}"
             '<div class="snapshot-title-url">'
             '<a class="snapshot-original-url" href="{}">{}<code style="user-select: all;">{}</code></a>'
             "</div>",
             detail_url,
+            format_html('<strong class="snapshot-delete-status">{}</strong>', "Deleting — verifying file removal")
+            if obj.status == Snapshot.DELETING_STATE
+            else "",
             title_html,
             url_raw or obj.url,
             favicon_html,
@@ -1535,25 +1540,14 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.action(
         description="🗑️ Delete",
+        permissions=["delete"],
     )
     def delete_snapshots(self, request, queryset):
-        """Delete snapshots in a single transaction to avoid SQLite concurrency issues."""
-        from django.db import transaction
-
-        total = queryset.count()
-
-        # Get list of IDs to delete first (outside transaction)
-        ids_to_delete = list(queryset.values_list("pk", flat=True))
-
-        # Delete everything in a single atomic transaction
-        with transaction.atomic():
-            deleted_count, _ = Snapshot.objects.filter(pk__in=ids_to_delete).delete()
-
+        """Keep the rows until the deletion worker removes and verifies their files."""
+        total = queryset.update(status=Snapshot.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
         messages.success(
             request,
-            mark_safe(
-                f"Successfully deleted {total} Snapshots ({deleted_count} total objects including related records). Don't forget to scrub URLs from import logs (data/sources) and error logs (data/logs) if needed.",
-            ),
+            f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
         )
 
     @admin.action(

archivebox/core/migrations/0058_snapshot_deleting_status.py

@@ -0,0 +1,28 @@
+# Generated by Django 6.1 on 2026-09-30 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0057_repair_migrated_process_links"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="snapshot",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("queued", "Queued"),
+                    ("started", "Started"),
+                    ("paused", "Paused"),
+                    ("sealed", "Sealed"),
+                    ("deleting", "Deleting"),
+                ],
+                db_index=True,
+                default="queued",
+                max_length=15,
+            ),
+        ),
+    ]

archivebox/core/models.py

@@ -1,6 +1,7 @@
 __package__ = "archivebox.core"
 
 import json
+import logging
 import os
 import uuid
 from collections.abc import Iterable, Mapping, Sequence
@@ -175,6 +176,19 @@ class SnapshotTag(models.Model):
 class SnapshotQuerySet(models.QuerySet):
     """Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
 
+    def delete(self):
+        # Finish the SELECT before filesystem work, so no read cursor or SQLite
+        # transaction is held while deleting remote archive payloads.
+        snapshots = list(self.select_related("crawl__created_by"))
+        total = 0
+        counts = {}
+        for snapshot in snapshots:
+            deleted, details = snapshot.delete(using=self.db)
+            total += deleted
+            for label, count in details.items():
+                counts[label] = counts.get(label, 0) + count
+        return total, counts
+
     def bulk_create(self, objs, *args, **kwargs):
         objs = list(objs)
         missing_crawl_ids = set()
@@ -541,6 +555,7 @@ class SnapshotManager(models.Manager.from_queryset(SnapshotQuerySet)):  # ty: ig
 
 class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats, ModelWithQueue):
     BROWSER_EXTENSION_UPLOAD_HOOK_NAME = "on_Snapshot__archivebox_browser_extension_upload"
+    DELETING_STATE = "deleting"
 
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -574,7 +589,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     )
     retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
     status = ModelWithQueue.StatusField(
-        choices=ModelWithQueue.StatusChoices,
+        choices=[*ModelWithQueue.StatusChoices.choices, (DELETING_STATE, "Deleting")],
         default=ModelWithQueue.StatusChoices.QUEUED,
     )
     config = models.JSONField(default=dict, null=False, blank=False, editable=True)
@@ -661,6 +676,47 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def __str__(self):
         return f"[{self.id}] {self.url[:64]}"
 
+    def output_paths_for_delete(self) -> tuple[Path, ...]:
+        current = self.get_storage_path_for_version(self.fs_version)
+        paths = [Path(self.output_dir), current, CONSTANTS.ARCHIVE_DIR / self.timestamp]
+        if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
+            paths.append(current.with_name(str(uuid.UUID(hex=self.id.hex))))
+        if self.crawl_id:
+            domain_dir = self.crawl.output_dir / CONSTANTS.SNAPSHOTS_DIR_NAME / self.extract_domain_from_url(self.url)
+            paths.extend((domain_dir / str(self.id), domain_dir / str(uuid.UUID(hex=self.id.hex))))
+        return tuple(dict.fromkeys(paths))
+
+    def delete(self, using=None, keep_parents=False):
+        from django.db import connections
+
+        using = using or self._state.db or "default"
+        if connections[using].in_atomic_block:
+            raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
+        type(self).objects.using(using).filter(pk=self.pk).update(status=self.DELETING_STATE, retry_at=RETRY_AT_MAX)
+        self.status = self.DELETING_STATE
+        from archivebox.machine.models import Process
+
+        process_ids = self.archiveresult_set.filter(process__status=Process.StatusChoices.RUNNING).values_list("process_id", flat=True)
+        for process in list(Process.objects.using(using).filter(pk__in=process_ids)):
+            process.kill_tree()
+            if process.is_running:
+                raise OSError(f"Snapshot output writer is still running: {process.pk}")
+        paths = self.output_paths_for_delete()
+        self.delete_output_paths(paths)
+        # Cascaded ArchiveResults share this directory. Their normal post-commit
+        # cleanup would repeat filesystem work and update an already-deleted row.
+        self._output_files_deleted = True
+        return super().delete(using=using, keep_parents=keep_parents)
+
+    @classmethod
+    def delete_requested(cls, *, batch_size=100):
+        pending = list(cls.objects.filter(status=cls.DELETING_STATE).select_related("crawl__created_by")[:batch_size])
+        for snapshot in pending:
+            try:
+                snapshot.delete()
+            except OSError:
+                logging.getLogger(__name__).exception("Snapshot %s file deletion failed; keeping its database row", snapshot.pk)
+
     @classmethod
     def crawl_count_subquery(cls, *, status: str | None = None, outer_ref: str = "pk") -> QuerySet:
         """Return a scalar subquery counting Snapshots for one outer Crawl."""

archivebox/core/snapshot_status.py

@@ -6,7 +6,7 @@ from django.db.models import QuerySet
 def snapshot_status_values() -> tuple[str, ...]:
     from archivebox.core.models import Snapshot
 
-    return tuple(Snapshot.StatusChoices.values)
+    return tuple(value for value, _label in Snapshot._meta.get_field("status").choices)
 
 
 def normalize_snapshot_status(status: str | None) -> str | None:

archivebox/machine/models.py

@@ -255,40 +255,40 @@ class Machine(ModelWithHealthStats):
                 **get_vm_info(),
                 stats=get_host_stats(),
             )
-        else:
-            if timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
-                for key, value in {
-                    "hostname": socket.gethostname(),
-                    **get_os_info(),
-                    **get_vm_info(),
-                    "stats": get_host_stats(),
-                }.items():
-                    setattr(machine, key, value)
-                machine.save(
-                    update_fields=[
-                        "hostname",
-                        "hw_in_docker",
-                        "hw_in_vm",
-                        "hw_manufacturer",
-                        "hw_product",
-                        "hw_uuid",
-                        "os_arch",
-                        "os_family",
-                        "os_platform",
-                        "os_release",
-                        "os_kernel",
-                        "stats",
-                        "modified_at",
-                    ],
-                )
-        machine = cls._sanitize_config(machine)
-        # Reconcile once, on the first Machine.current() call in this process.
+        needs_recheck = timezone.now() >= machine.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL)
+        # Reconcile before any save can mirror stale DB config over file edits.
         try:
             from archivebox.config.collection import sync_machine_and_file
 
             sync_machine_and_file(machine)
         except Exception:
             pass
+        if needs_recheck:
+            for key, value in {
+                "hostname": socket.gethostname(),
+                **get_os_info(),
+                **get_vm_info(),
+                "stats": get_host_stats(),
+            }.items():
+                setattr(machine, key, value)
+            machine.save(
+                update_fields=[
+                    "hostname",
+                    "hw_in_docker",
+                    "hw_in_vm",
+                    "hw_manufacturer",
+                    "hw_product",
+                    "hw_uuid",
+                    "os_arch",
+                    "os_family",
+                    "os_platform",
+                    "os_release",
+                    "os_kernel",
+                    "stats",
+                    "modified_at",
+                ],
+            )
+        machine = cls._sanitize_config(machine)
         # Publish only after initialization is complete.
         _CURRENT_MACHINE = machine
         return machine

archivebox/personas/models.py

@@ -74,8 +74,12 @@ def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, person
         derived["COOKIES_FILE"] = str(cookies_path)
 
     auth_path = persona_dir / "auth.json"
-    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
-        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+    if "AUTH_STORAGE_FILE" not in derived:
+        try:
+            if auth_path.stat().st_size > 0:
+                derived["AUTH_STORAGE_FILE"] = str(auth_path)
+        except FileNotFoundError:
+            pass
 
     derived["ACTIVE_PERSONA"] = name
     return derived
@@ -156,9 +160,12 @@ class Persona(ModelWithConfig):
 
     @property
     def AUTH_STORAGE_FILE(self) -> str:
-        """Derived path to auth.json for this persona (if it exists)."""
+        """Derived path to a nonempty auth.json export for this persona."""
         auth_path = self.path / "auth.json"
-        return str(auth_path) if auth_path.exists() else ""
+        try:
+            return str(auth_path) if auth_path.stat().st_size > 0 else ""
+        except FileNotFoundError:
+            return ""
 
     def get_derived_config(self) -> dict:
         """
@@ -315,20 +322,9 @@ class Persona(ModelWithConfig):
             if chrome_binary:
                 (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # Hooks derive CHROME_USER_DATA_DIR/CHROME_DOWNLOADS_DIR from
-            # PERSONAS_DIR + ACTIVE_PERSONA. Point PERSONAS_DIR at the
-            # per-crawl runtime root here so CHROME_ISOLATION=crawl never
-            # leaks or reuses the template profile while keeping Chrome path
-            # derivation centralized in the Chrome plugin helpers.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        # Hooks derive Chrome profile paths from the runtime persona root.
+        # Use the same optional auth discovery as the template persona.
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
         crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
@@ -358,18 +354,7 @@ class Persona(ModelWithConfig):
         if chrome_binary:
             (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
 
-        return {
-            # See prepare_runtime_for_crawl(): snapshot isolation changes the
-            # persona root, not individual CHROME_* config keys, so standalone
-            # Chrome hooks and ArchiveBox-driven hooks resolve paths the same way.
-            "PERSONAS_DIR": str(runtime_root.parent),
-            "ACTIVE_PERSONA": self.name,
-            **{
-                key: str(runtime_root / filename)
-                for key, filename in (("COOKIES_FILE", "cookies.txt"), ("AUTH_STORAGE_FILE", "auth.json"))
-                if (runtime_root / filename).is_file()
-            },
-        }
+        return derive_persona_config(name=self.name, config={}, persona_dir=runtime_root)
 
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)

archivebox/services/runner.py

@@ -65,6 +65,7 @@ from archivebox.services.resource_admission import RESOURCE_RECHECK_SECONDS, res
 from archivebox.config.common import (
     ArchiveBoxBaseConfig,
     normalize_runtime_config,
+    _is_cookie_file_config_key,
     _plugin_enabled_config_keys,
 )
 from archivebox.misc.db import run_db_analyze_batch
@@ -738,7 +739,14 @@ class CrawlRunner:
             current_process.save(update_fields=["iface", "machine", "modified_at"])
         self.persona = self.crawl.resolve_persona()
         self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
-        self.derived_config = dict(machine.config or {})
+        # The merged user payload owns auth paths and binary selections. The
+        # install phase publishes resolved binaries; stale machine paths must
+        # not override explicit runtime settings before that phase runs.
+        self.derived_config = {
+            key: value
+            for key, value in (machine.config or {}).items()
+            if not _is_cookie_file_config_key(key) and (not key.endswith("_BINARY") or value == self.base_config.get(key))
+        }
         self.crawl_output_dir = str(self.crawl.output_dir)
         if self.persona:
             self.base_config.update(
@@ -1989,6 +1997,13 @@ def run_pending_crawls(
         raise_if_shutdown_requested()
         now_monotonic = time.monotonic()
         if crawl_id is None and now_monotonic - last_retention_at >= (60.0 if daemon else 1.0):
+            if not daemon:
+                from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+
+                supervisor = get_existing_supervisord_process(quiet=True)
+                deletion_worker = get_worker(supervisor, "worker_snapshot_delete") if supervisor else None
+                if not deletion_worker or deletion_worker.get("statename") not in ("STARTING", "RUNNING"):
+                    Snapshot.delete_requested(batch_size=100)
             for model in (ArchiveResult, Snapshot, Crawl, Process):
                 # Keep the tight scheduler loop anchored on indexed delete_at
                 # columns only. Backfilling missing delete_at values has to read

archivebox/tests/test_cli_persona.py

@@ -50,12 +50,14 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
     """Import an actual Chromium-created session and archive its private page."""
     import hashlib
     import json
+    import shlex
     import subprocess
 
     from werkzeug.wrappers import Response
     from archivebox.tests.conftest import resolve_abxpkg_chrome_env
 
     browser_env = resolve_abxpkg_chrome_env(tmp_path / "lib")
+    browser_env["ABXPKG_LIB_DIR"] = str(tmp_path / "lib")
     source = tmp_path / "browser"
     httpserver.expect_request("/login").respond_with_data(
         '<html><title>Signed in</title><script>localStorage.setItem("persona-preference", "forest-green")</script>Signed in</html>',
@@ -90,6 +92,7 @@ const puppeteer = require(process.argv[1]);
         const page = await browser.newPage();
         await page.goto(process.argv[4]);
         console.log(await page.title());
+        console.log(await page.evaluate(() => localStorage.getItem('persona-preference')));
     } finally { await browser.close(); }
 })().catch(error => { console.error(error); process.exit(1); });
 """,
@@ -104,6 +107,7 @@ const puppeteer = require(process.argv[1]);
     )
     assert seeded.returncode == 0, seeded.stderr
     assert "Signed in" in seeded.stdout
+    assert "forest-green" in seeded.stdout
     preferences = source / "Profile 2" / "Preferences"
     original_preferences = preferences.read_bytes()
 
@@ -161,6 +165,9 @@ const puppeteer = require(process.argv[1]);
         env={**browser_env, "CHROME_SANDBOX": "false"},
     )
     assert archived.returncode == 0, archived.stderr
+    launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
+    assert len(launch_commands) == 1
+    assert shlex.split(launch_commands[0].read_text().splitlines()[1])[0] == browser_env["CHROME_BINARY"]
     outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
     assert len(outputs) == 1
     html = outputs[0].read_text()

archivebox/tests/test_cli_server.py

@@ -448,7 +448,11 @@ def test_port_compatibility_does_not_add_listeners_outside_docker_defaults(tmp_p
         check=True,
     )
     workers = json.loads(result.stdout)
-    assert [worker["name"] for worker in workers] == ["worker_runserver" if debug else "worker_daphne", "worker_runner"]
+    assert [worker["name"] for worker in workers] == [
+        "worker_runserver" if debug else "worker_daphne",
+        "worker_runner",
+        "worker_snapshot_delete",
+    ]
     command = shlex.split(workers[0]["command"])
     assert "--endpoint" not in command
     assert (f"127.0.0.1:{port}" if debug else f"--port={port}") in command

archivebox/tests/test_core_config.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 import pytest
 from pydantic import ValidationError
@@ -95,3 +96,40 @@ def test_resolving_scoped_config_does_not_mutate_process_environment(tmp_path):
             os.environ.pop("ABXPKG_LIB_DIR", None)
         else:
             os.environ["ABXPKG_LIB_DIR"] = previous_lib_dir
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE", "WGET_COOKIES_FILE", "YTDLP_COOKIES_FILE", "GALLERYDL_COOKIES_FILE"])
+def test_cookie_paths_remain_collection_relative_after_runtime_overlays(key, tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: "auth/cookies.txt"})
+    assert str(config[key]) == str(CONSTANTS.DATA_DIR / "auth/cookies.txt")
+    runtime = config.for_crawl_runtime(runtime_overrides={key: "other/cookies.txt"})
+    assert runtime[key] == str(CONSTANTS.DATA_DIR / "other/cookies.txt")
+    absolute = tmp_path / "absolute-cookies.txt"
+    config = ArchiveBoxConfig(**{key: str(absolute)})
+    assert str(config[key]) == str(absolute)
+
+
+@pytest.mark.parametrize("key", ["COOKIES_FILE", "AUTH_STORAGE_FILE"])
+@pytest.mark.parametrize("value", ["", "  ", None])
+def test_empty_cookie_path_disables_import(key, value):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(**{key: value})
+    assert config[key] is None
+    assert key not in config.for_crawl_runtime()
+
+
+def test_cookie_runtime_path_preserves_configured_symlink(tmp_path):
+    from archivebox.config.common import ArchiveBoxConfig
+
+    export = tmp_path / "export.txt"
+    export.write_text("# Netscape HTTP Cookie File\n")
+    configured = tmp_path / "cookies.txt"
+    configured.symlink_to(export)
+    config = ArchiveBoxConfig(COOKIES_FILE=configured)
+    assert str(config.COOKIES_FILE) == str(configured)
+    runtime = config.for_crawl_runtime()
+    assert runtime["COOKIES_FILE"] == str(configured)
+    assert Path(runtime["COOKIES_FILE"]).read_bytes() == export.read_bytes()

archivebox/tests/test_issue_1896.py

@@ -0,0 +1,153 @@
+"""Regression coverage for collection cookie paths and config edits on upgrade."""
+
+import json
+import sqlite3
+from contextlib import closing
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import pytest
+
+from archivebox.config.configset import read_ini_config
+from archivebox.tests.conftest import run_archivebox_cmd
+
+
+@pytest.mark.parametrize("contents,explicit", [("", True), ("not JSON", True), ("not JSON", False)])
+def test_add_rejects_invalid_persona_auth_with_file_path(initialized_archive, httpserver, contents, explicit):
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.write_text(contents)
+    if explicit:
+        run_archivebox_cmd(
+            ["config", "--set", "AUTH_STORAGE_FILE=personas/Default/auth.json"],
+            cwd=initialized_archive,
+            check=True,
+        )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 1, result.stdout + result.stderr
+    logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    assert logs
+    assert f"Invalid JSON cookie export: {auth_file}" in "\n".join(log.read_text() for log in logs)
+    assert not list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert auth_file.read_text() == contents
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+@pytest.mark.parametrize("isolation", ["crawl", "snapshot"])
+def test_add_with_empty_autodiscovered_persona_auth(initialized_archive, httpserver, cookie_file_exists, isolation):
+    """An empty auth placeholder must not block a crawl or hide Netscape cookies."""
+    persona_dir = initialized_archive / "personas" / "Default"
+    persona_dir.mkdir(parents=True, exist_ok=True)
+    auth_file = persona_dir / "auth.json"
+    auth_file.touch()
+    cookies_file = persona_dir / "cookies.txt"
+    cookies_file.write_text(
+        "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n" if cookie_file_exists else "",
+    )
+    httpserver.expect_request("/").respond_with_data(
+        '<html><title>Empty persona auth</title><body><output id="cookies"></output>'
+        '<script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false", "CHROME_ISOLATION": isolation},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    assert ("issue1896=imported" in outputs[0].read_text()) == cookie_file_exists
+    assert auth_file.read_bytes() == b""
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT cmd, env, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for cmd, env, exit_code in hook_rows:
+            hook_env = json.loads(env)
+            assert not hook_env.get("AUTH_STORAGE_FILE")
+            expected_cookies = cookies_file if isolation == "crawl" else Path(hook_env["PERSONAS_DIR"]) / "Default" / "cookies.txt"
+            assert hook_env["COOKIES_FILE"] == str(expected_cookies)
+            hook_name = Path(json.loads(cmd)[0]).name
+            # The crawl wait hook explicitly skips when each snapshot owns Chrome.
+            assert exit_code == (10 if isolation == "snapshot" and hook_name == "on_CrawlSetup__91_chrome_wait.js" else 0)
+
+
+@pytest.mark.parametrize("cookie_file_exists", [True, False])
+def test_add_with_collection_relative_cookies(initialized_archive, httpserver, cookie_file_exists):
+    """Real crawl hooks must read cookies from the collection, not their cwd."""
+    if cookie_file_exists:
+        (initialized_archive / "cookies.txt").write_text(
+            "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t2147483647\tissue1896\timported\n",
+        )
+    httpserver.expect_request("/").respond_with_data(
+        "<html><title>Issue 1896</title><body><p>Collection cookie import regression: "
+        "this page must still be archived when the optional cookie file is absent.</p>"
+        '<output id="cookies"></output><script>document.getElementById("cookies").textContent = document.cookie;</script></body></html>',
+        content_type="text/html",
+    )
+    run_archivebox_cmd(
+        ["config", "--set", "COOKIES_FILE=cookies.txt"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    result = run_archivebox_cmd(
+        ["add", "--plugins=dom", httpserver.url_for("/").replace("localhost", "127.0.0.1")],
+        cwd=initialized_archive,
+        timeout=180,
+        env={"CHROME_SANDBOX": "false"},
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
+    assert len(outputs) == 1, result.stdout + result.stderr
+    html = outputs[0].read_text()
+    assert ("issue1896=imported" in html) == cookie_file_exists
+    launch_logs = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/*.stderr.log"))
+    logs = "\n".join(log.read_text() for log in launch_logs)
+    if cookie_file_exists:
+        assert f"from {initialized_archive / 'cookies.txt'}" in logs
+    else:
+        assert "Cookies file not found" in logs
+        assert "continuing without cookie import" in logs
+    with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+        assert db.execute("SELECT status FROM core_snapshot").fetchall() == [("sealed",)]
+        assert db.execute("SELECT status FROM core_archiveresult WHERE plugin='dom'").fetchall() == [("succeeded",)]
+        hook_rows = db.execute("SELECT env, status, exit_code FROM machine_process WHERE process_type='hook'").fetchall()
+        assert hook_rows
+        for env, status, exit_code in hook_rows:
+            assert json.loads(env)["COOKIES_FILE"] == str(initialized_archive / "cookies.txt")
+            assert (status, exit_code) == ("exited", 0)
+
+
+@pytest.mark.parametrize("enabled_key", ["USE_CHROME", "CHROME_ENABLED"])
+@pytest.mark.parametrize("startup_save", ["binary_sanitation", "host_refresh"])
+def test_init_preserves_edited_use_chrome_before_startup_save(initialized_archive, enabled_key, startup_save):
+    """Startup saves must not overwrite a newer file before config sync."""
+    config_path = initialized_archive / "ArchiveBox.conf"
+    config_path.write_text(config_path.read_text() + f"\n[SERVER_CONFIG]\n{enabled_key} = True\n")
+    run_archivebox_cmd(
+        ["config", "--set", f"WGET_BINARY={initialized_archive / 'removed-wget'}"]
+        if startup_save == "binary_sanitation"
+        else ["config", "--set", "TIMEOUT=37"],
+        cwd=initialized_archive,
+        check=True,
+    )
+    if startup_save == "host_refresh":
+        with closing(sqlite3.connect(initialized_archive / "index.sqlite3")) as db:
+            db.execute("UPDATE machine_machine SET modified_at=?", [(datetime.now(timezone.utc) - timedelta(days=8)).isoformat()])
+            db.commit()
+    config_path.write_text(config_path.read_text().replace(f"{enabled_key} = True", f"{enabled_key} = False"))
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    run_archivebox_cmd(["init"], cwd=initialized_archive, timeout=120, check=True)
+    assert read_ini_config(config_path)[enabled_key] == "False"
+    result = run_archivebox_cmd(["config", "--get", "CHROME_ENABLED"], cwd=initialized_archive)
+    assert result.returncode == 0, result.stderr
+    assert "CHROME_ENABLED = false" in result.stdout

archivebox/tests/test_snapshot_delete_worker.py

@@ -0,0 +1,114 @@
+"""Exercise filesystem failures and worker isolation with a real server."""
+
+import json
+import os
+import signal
+import time
+from pathlib import Path
+
+import pytest
+
+from archivebox.tests.conftest import (
+    cli_env,
+    get_free_port,
+    kill_processes_for_data_dir,
+    run_archivebox_cmd,
+    start_archivebox_server,
+    stop_archivebox_process,
+    wait_for_log,
+)
+from archivebox.tests.test_orm_helpers import use_archivebox_db
+
+
+@pytest.mark.django_db(transaction=True)
+def test_failed_and_paused_deletion_worker_does_not_block_captures(initialized_archive, recursive_test_site):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.machine.models import Process
+
+    port = get_free_port()
+    env = cli_env(
+        live=True,
+        server=True,
+        port=port,
+        PLUGINS="title",
+        SAVE_TITLE="True",
+        SEARCH_BACKEND_ENGINE="ripgrep",
+    )
+    prepared = run_archivebox_cmd(
+        [
+            "manage",
+            "shell",
+            "-c",
+            "import json; from archivebox.crawls.models import Crawl; "
+            "from archivebox.core.models import Snapshot, ArchiveResult; "
+            "from archivebox.base_models.models import get_or_create_system_user_pk; "
+            "c=Crawl.objects.create(urls='https://example.com/delete',created_by_id=get_or_create_system_user_pk(),status='sealed'); "
+            "s=Snapshot.objects.create(url='https://example.com/delete',crawl=c,status='deleting'); "
+            "p=s.output_dir/'wget'; p.mkdir(parents=True,exist_ok=True); "
+            "(p/'index.html').write_text('keep this until cleanup succeeds'); "
+            "ArchiveResult.objects.create(snapshot=s,plugin='wget',hook_name='on_Snapshot__06_wget',status='succeeded'); "
+            "p.chmod(0o500); print(json.dumps({'id':str(s.pk),'path':str(p)}))",
+        ],
+        cwd=initialized_archive,
+        env=env,
+    )
+    assert prepared.returncode == 0, prepared.stderr or prepared.stdout
+    fixture = json.loads(prepared.stdout.strip().splitlines()[-1])
+    protected = Path(fixture["path"])
+    worker_log = initialized_archive / "logs" / "worker_snapshot_delete.log"
+    server = None
+    deletion_pid = None
+    try:
+        server = start_archivebox_server(initialized_archive, port=port, env=env, log_name="delete-worker-server.log")
+        wait_for_log(worker_log, "PermissionError", timeout=30)
+        with use_archivebox_db(initialized_archive):
+            deletion = Process.objects.get(worker_type="worker_snapshot_delete", status="running")
+            runner = Process.objects.get(worker_type="worker_runner", status="running")
+            assert deletion.pid != runner.pid
+            deletion_pid = deletion.pid
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+            assert ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert (protected / "index.html").is_file()
+        # Suspend the actual cleanup process. The scheduler and its real title
+        # hook must still finish a new capture without waiting for this worker.
+        os.kill(deletion_pid, signal.SIGSTOP)
+        added = run_archivebox_cmd(
+            ["add", "--bg", "--plugins=title", recursive_test_site["root_url"]],
+            cwd=initialized_archive,
+            env=env,
+        )
+        assert added.returncode == 0, added.stderr or added.stdout
+        deadline = time.monotonic() + 60
+        with use_archivebox_db(initialized_archive):
+            while time.monotonic() < deadline:
+                captured = Snapshot.objects.filter(url=recursive_test_site["root_url"], status="sealed").first()
+                if captured is not None:
+                    break
+                assert server.poll() is None
+                time.sleep(0.1)
+            else:
+                pytest.fail((initialized_archive / "logs" / "worker_runner.log").read_text()[-12000:])
+            result = ArchiveResult.objects.get(snapshot=captured, plugin="title", status="succeeded")
+            assert result.output_files
+            assert captured.title == "Root"
+            assert Snapshot.objects.filter(pk=fixture["id"], status="deleting").exists()
+        protected.chmod(0o700)
+        os.kill(deletion_pid, signal.SIGCONT)
+        deadline = time.monotonic() + 75
+        with use_archivebox_db(initialized_archive):
+            while Snapshot.objects.filter(pk=fixture["id"]).exists() and time.monotonic() < deadline:
+                time.sleep(0.1)
+            assert not Snapshot.objects.filter(pk=fixture["id"]).exists(), worker_log.read_text()
+            assert not ArchiveResult.objects.filter(snapshot_id=fixture["id"]).exists()
+        assert not protected.parent.exists()
+    finally:
+        if protected.exists():
+            protected.chmod(0o700)
+        if deletion_pid is not None:
+            try:
+                os.kill(deletion_pid, signal.SIGCONT)
+            except ProcessLookupError:
+                pass
+        if server is not None:
+            stop_archivebox_process(server)
+        kill_processes_for_data_dir(initialized_archive)

archivebox/tests/test_ui_admin_crawl.py

@@ -343,6 +343,7 @@ def test_discovered_snapshots_inherit_current_crawl_tags(crawl):
     assert set(created[0].tags.values_list("name", flat=True)) == {"midcrawl", "discovered"}
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, admin_user):
     crawl = Crawl.objects.create(
         urls="https://example.com/remove-me",
@@ -352,6 +353,9 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
         crawl=crawl,
         url="https://example.com/remove-me",
     )
+    output_dir = snapshot.output_dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "captured.txt").write_text("remove this capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -362,11 +366,13 @@ def test_crawl_admin_delete_snapshot_action_removes_snapshot_and_url(client, adm
     assert response.status_code == 200
     assert response.json()["ok"] is True
     assert not Snapshot.objects.filter(pk=snapshot.pk).exists()
+    assert not output_dir.exists()
 
     crawl.refresh_from_db()
     assert "https://example.com/remove-me" not in crawl.urls
 
 
+@pytest.mark.django_db(transaction=True)
 def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(client, admin_user):
     crawl = Crawl.objects.create(
         urls=("https://cdn.example.com/asset.js\nhttps://cdn.example.com/second.js\nhttps://example.com/root"),
@@ -382,6 +388,12 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
         url="https://example.com/root",
         status=Snapshot.StatusChoices.SEALED,
     )
+    removed_dir = queued_snapshot.output_dir
+    removed_dir.mkdir(parents=True, exist_ok=True)
+    (removed_dir / "captured.txt").write_text("excluded capture")
+    preserved_file = preserved_snapshot.output_dir / "captured.txt"
+    preserved_file.parent.mkdir(parents=True, exist_ok=True)
+    preserved_file.write_text("preserved capture")
 
     client.force_login(admin_user)
     response = client.post(
@@ -401,6 +413,8 @@ def test_crawl_admin_exclude_domain_action_prunes_urls_and_pending_snapshots(cli
     assert "https://example.com/root" in crawl.urls
     assert not Snapshot.objects.filter(pk=queued_snapshot.pk).exists()
     assert Snapshot.objects.filter(pk=preserved_snapshot.pk).exists()
+    assert not removed_dir.exists()
+    assert preserved_file.read_text() == "preserved capture"
 
 
 def test_snapshot_from_json_trims_markdown_suffixes_on_discovered_urls(crawl):

archivebox/tests/test_ui_admin_snapshot.py

@@ -26,6 +26,65 @@ pytestmark = pytest.mark.django_db(transaction=True)
 REPO_ROOT = Path(__file__).resolve().parents[2]
 
 
+def test_bulk_snapshot_delete_keeps_rows_until_worker_verifies_files(client, admin_user, crawl):
+    from archivebox.core.models import ArchiveResult, Snapshot
+    from archivebox.services.runner import run_pending_crawls
+
+    snapshots = [Snapshot.objects.create(url=f"https://example.com/delete/{i}", crawl=crawl) for i in range(2)]
+    output_dirs = [snapshot.output_dir for snapshot in snapshots]
+    for snapshot, output_dir in zip(snapshots, output_dirs):
+        (output_dir / "wget").mkdir(parents=True, exist_ok=True)
+        (output_dir / "wget" / "index.html").write_text("real saved output")
+        ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    ids = [snapshot.pk for snapshot in snapshots]
+    client.force_login(admin_user)
+    response = client.post(
+        reverse("admin:core_snapshot_changelist"),
+        {"action": "delete_snapshots", ACTION_CHECKBOX_NAME: [str(pk) for pk in ids]},
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+    assert response.status_code == 302
+    assert Snapshot.objects.filter(pk__in=ids, status="deleting").count() == 2
+    assert ArchiveResult.objects.filter(snapshot_id__in=ids).count() == 2
+    assert all((path / "wget" / "index.html").is_file() for path in output_dirs)
+    pending = client.get(reverse("admin:core_snapshot_changelist"), HTTP_HOST=ADMIN_TEST_HOST)
+    assert pending.status_code == 200
+    assert "Deleting" in pending.content.decode()
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk__in=ids).exists()
+    assert not ArchiveResult.objects.filter(snapshot_id__in=ids).exists()
+    assert not any(path.exists() for path in output_dirs)
+
+
+def test_snapshot_delete_preserves_rows_when_real_filesystem_cleanup_fails(snapshot):
+    from archivebox.core.models import ArchiveResult, Snapshot
+
+    output_dir = snapshot.output_dir
+    protected = output_dir / "wget"
+    protected.mkdir(parents=True, exist_ok=True)
+    saved_file = protected / "index.html"
+    saved_file.write_text("must not lose the row before this file is removed")
+    result = ArchiveResult.objects.create(snapshot=snapshot, plugin="wget", hook_name="on_Snapshot__06_wget", status="succeeded")
+    snapshot_id = snapshot.pk
+    protected.chmod(0o500)
+    try:
+        with pytest.raises(PermissionError):
+            Snapshot.objects.filter(pk=snapshot_id).delete()
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert ArchiveResult.objects.filter(pk=result.pk).exists()
+        assert saved_file.is_file()
+        from archivebox.services.runner import run_pending_crawls
+
+        run_pending_crawls(maintenance_only=True)
+        assert Snapshot.objects.filter(pk=snapshot_id, status="deleting").exists()
+        assert saved_file.is_file()
+    finally:
+        protected.chmod(0o700)
+    run_pending_crawls(maintenance_only=True)
+    assert not Snapshot.objects.filter(pk=snapshot_id).exists()
+    assert not output_dir.exists()
+
+
 def test_failed_hook_partial_file_does_not_become_successful_card(snapshot, cached_abxpkg_lib_dir):
     partial = Path(snapshot.output_dir) / "title" / "partial.html"
     partial.parent.mkdir(parents=True, exist_ok=True)

archivebox/tests/test_ui_config_views.py

@@ -7,7 +7,7 @@ from archivebox.plugins import views as plugin_views
 from archivebox.core import views as core_views
 from archivebox.config import CONSTANTS
 from archivebox.machine.models import Machine
-from archivebox.tests.conftest import install_real_binary
+from archivebox.tests.conftest import ADMIN_TEST_HOST, install_real_binary
 
 
 pytestmark = pytest.mark.django_db
@@ -75,18 +75,42 @@ def test_binaries_list_view_uses_db_version_and_hides_youtube_dl_alias(admin_req
 
     assert len(context["table"]["Binary Name"]) == 1
     assert str(context["table"]["Binary Name"][0].link_item) == "yt-dlp"
-    assert context["table"]["Found Version"][0] == f"✅ {binary.version}"
+    assert context["table"]["Recorded Version"][0] == binary.version
     assert context["table"]["Provided By"][0] == binary.binprovider
-    assert context["table"]["Found Abspath"][0] == binary.abspath
+    assert context["table"]["Recorded Abspath"][0] == binary.abspath
 
 
 def test_binaries_list_view_only_shows_persisted_records(admin_request):
     context = config_views.binaries_list_view.__wrapped__(admin_request("/admin/environment/binaries/"))
 
     assert context["table"]["Binary Name"] == []
-    assert context["table"]["Found Version"] == []
+    assert context["table"]["Recorded Version"] == []
     assert context["table"]["Provided By"] == []
-    assert context["table"]["Found Abspath"] == []
+    assert context["table"]["Recorded Abspath"] == []
+
+
+@pytest.mark.django_db(transaction=True)
+def test_binaries_page_labels_removed_installation_as_recorded_metadata(admin_client, machine, hermetic_lib_dir):
+    from pathlib import Path
+
+    binary = install_real_binary("node", machine=machine)
+    binary_path = Path(binary.abspath)
+    assert binary_path.is_relative_to(hermetic_lib_dir)
+    assert binary_path.is_symlink()
+    binary_path.unlink()
+    assert not binary_path.exists()
+
+    response = admin_client.get("/admin/environment/binaries/", HTTP_HOST=ADMIN_TEST_HOST)
+    assert response.status_code == 200
+    html = response.content.decode()
+    assert "/admin/environment/binaries/node/" in html
+    assert "Recorded Version" in html
+    assert "Recorded Abspath" in html
+    assert binary.version in html
+    assert str(binary_path) in html
+    assert f"✅ {binary.version}" not in html
+    binary.refresh_from_db()
+    assert binary.abspath == str(binary_path)
 
 
 @pytest.mark.django_db(transaction=True)

archivebox/workers/management/commands/snapshot_delete.py

@@ -0,0 +1,23 @@
+import time
+
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    help = "Remove queued Snapshot files before deleting their database rows."
+
+    def handle(self, *args, **kwargs):
+        from archivebox.config import CONSTANTS
+        from archivebox.core.models import Snapshot
+        from archivebox.machine.models import Process
+
+        current = Process.current()
+        current.mark_running(
+            process_type=Process.TypeChoices.WORKER,
+            worker_type="worker_snapshot_delete",
+            pwd=str(CONSTANTS.DATA_DIR),
+        )
+        while True:
+            Snapshot.delete_requested(batch_size=100)
+            current.heartbeat()
+            time.sleep(60)

archivebox/workers/supervisord_util.py

@@ -301,6 +301,22 @@ def RUNNER_WORKER():
     }
 
 
+def SNAPSHOT_DELETE_WORKER():
+    command = archivebox_cmd("manage", "snapshot_delete")
+    return {
+        "name": "worker_snapshot_delete",
+        "command": _shell_join(command),
+        "autostart": "false",
+        "autorestart": "true",
+        "environment": _archivebox_worker_environment(command, PYTHONUNBUFFERED="1"),
+        "stopasgroup": "true",
+        "killasgroup": "true",
+        "stopwaitsecs": "30",
+        "stdout_logfile": "logs/worker_snapshot_delete.log",
+        "redirect_stderr": "true",
+    }
+
+
 def RUNNER_ONCE_WORKER(args, name="worker_runner_once"):
     command = archivebox_cmd("run", "--no-stdin", *args)
     return {
@@ -1336,6 +1352,9 @@ def build_server_worker_plan(*, config, host: str, port: str, debug: bool, reloa
         bg_workers = [(RUNNER_WORKER(), False)]
         log_files = ["logs/worker_daphne.log", "logs/worker_runner.log"]
 
+    bg_workers.append((SNAPSHOT_DELETE_WORKER(), False))
+    log_files.append("logs/worker_snapshot_delete.log")
+
     sonic_worker = get_sonic_supervisord_worker_from_plugin(config)
     if sonic_worker is not None:
         try:

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.71",
+  "version": "0.9.72rc11",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.71"
+version = "0.9.72rc11"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,8 +79,8 @@ dependencies = [
     ### Binary/Package Management
     "abxbus==2.5.70",                 # direct imports only; version constrained by abx-dl -> abx-plugins
     "abxpkg==1.13.21",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins[opencode]==1.13.109", # optional AI runtime clients are owned by the plugin
-    "abx-dl==1.13.79",       # shared ArchiveBox downloader package
+    "abx-plugins[opencode]==1.13.114", # optional AI runtime clients are owned by the plugin
+    "abx-dl==1.13.82",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
     "browser-cookie3>=0.20.1",
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.71"
+current_version = "v0.9.72rc11"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-23T19:26:02.6709672Z"
+exclude-newer = "2026-09-25T22:18:50.337323517Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-28T19:26:01.670984315Z", span = "PT1S" }
+abxbus = { timestamp = "2026-09-30T22:18:49.337336942Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.13.79"
+version = "1.13.82"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f5/59/e178d330d244351f01026a2c261c7c0d8067c0efcbef9cf095635fa97579/abx_dl-1.13.79.tar.gz", hash = "sha256:8825b06570df48f3ecdd525e8fca09a2d8dac77b8ba059efaac7d0df845b578a", size = 97588, upload-time = "2026-09-28T19:23:47.984Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/42aefa8973feca5ccc27f5eb2af517a580cdf954dbc720569ae8cf35d8ba/abx_dl-1.13.82.tar.gz", hash = "sha256:03d1b87302818e03f6d763a64b3007de6183d5d529b01e589f3169540e9e511e", size = 97555, upload-time = "2026-09-30T22:17:20.811Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/60/6f/f7de1fd42bbcdf56769a7277ca261007525d2a482544fe03d509b3cc6805/abx_dl-1.13.79-py3-none-any.whl", hash = "sha256:3b9b8426da6c0fcd55583d0f26c037561b40982866c4a12f481c8f481fbe56b2", size = 102934, upload-time = "2026-09-28T19:23:49.287Z" },
+    { url = "https://files.pythonhosted.org/packages/18/c6/e6ca94e5b2a49ed9c9c313a995e5ded415c703c55b0090d898fb4cb71fb0/abx_dl-1.13.82-py3-none-any.whl", hash = "sha256:aa8fad25e2d0af2143b363b1c9cb2bfe581aec1eb30d331ce33d049572cf0197", size = 102935, upload-time = "2026-09-30T22:17:19.099Z" },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.13.109"
+version = "1.13.114"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/22/e236ab70ff3c24c3be7f511c202e9943b4acf7988ca180f6593a959cf190/abx_plugins-1.13.109.tar.gz", hash = "sha256:056556249fbe5e1aa5eca3cca84923fab9a4a1fcfd0d3f82c4911de97f2e49f5", size = 461059, upload-time = "2026-09-28T19:08:05.06Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/dd/44/b6dc97f8146dd54bc921465ff909142f77dec1cae93766a1c7a6c387df51/abx_plugins-1.13.114.tar.gz", hash = "sha256:ee9f6ef5ab4dd93b06d6faba0b000c12f98de166cf23d2b6dc1e0ad990c43c32", size = 461642, upload-time = "2026-09-30T21:49:04.533Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a4/c9/6331394242f8c5afa1fb03abfd805e509add8a7d03c14b9e126e6724af20/abx_plugins-1.13.109-py3-none-any.whl", hash = "sha256:5bdecde68a9bc2265a26b95205c9214a07e39328c981c8700234f577a57d0ba7", size = 673893, upload-time = "2026-09-28T19:08:06.808Z" },
+    { url = "https://files.pythonhosted.org/packages/1d/e6/fa22c699951d7f935d1ed6c06a509c1e259e441966f45569c9374bce480d/abx_plugins-1.13.114-py3-none-any.whl", hash = "sha256:5b9ea246d743f5b50d0510a5078915bc88a902a3765c4362ac6fe6bd0365a4ca", size = 674540, upload-time = "2026-09-30T21:49:06.268Z" },
 ]
 
 [package.optional-dependencies]
@@ -127,7 +127,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.71"
+version = "0.9.72rc11"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -227,8 +227,8 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.13.79" },
-    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.109" },
+    { name = "abx-dl", specifier = "==1.13.82" },
+    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.114" },
     { name = "abxbus", specifier = "==2.5.70" },
     { name = "abxpkg", specifier = "==1.13.21" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },