Bump release version to 0.9.35rc404

Supersedes #1818.

This keeps yt-dlp-specific player rendering out of ArchiveBox core. Core
now:

- exposes media type metadata to plugin card templates: `is_video`,
`is_audio`, `is_browser_playable`
- sorts yt-dlp media with browser-playable video/audio first
- prefers browser-playable yt-dlp outputs before thumbnails and
non-browser containers when choosing the primary card output
- avoids pinning resumed snapshots to stale selected hook maps, fixing a
pause/resume runner regression surfaced by CI
- makes `update --index-only` discover the configured search backend
even when `PLUGINS` is restricted for test/runtime isolation
- stabilizes CI-only recursive/takeover/public-UI tests that were
failing due external/heavy plugin coverage, hard-kill recovery races,
and transient full-server `/add/` startup responses

The actual click-to-load player UI lives in the companion plugin PR:
ArchiveBox/abx-plugins#34.

Verification:
```text
pytest archivebox/tests/test_ui_admin_snapshot.py -q
48 passed

pytest archivebox/tests/test_cli_update_reindex_snapshots.py -q
9 passed

pytest -xvs archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py --basetemp=tests/out --ignore=archivebox/pkgs
7 passed

pytest -xvs archivebox/tests/test_takeover_util.py --basetemp=tests/out --ignore=archivebox/pkgs
19 passed

pytest -xvs archivebox/tests/test_takeover_util.py::test_live_add_update_jobs_survive_server_and_cli_owner_exits --basetemp=tests/out --ignore=archivebox/pkgs
1 passed

pytest -xvs archivebox/tests/test_ui_public_snapshot.py --basetemp=tests/out --ignore=archivebox/pkgs
6 passed

pytest -q archivebox/tests/test_recursive_crawl.py::test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel --basetemp=tests/out --ignore=archivebox/pkgs
1 passed
```

.github/workflows/docs.yml

@@ -99,9 +99,10 @@ jobs:
           if [[ '${{ matrix.environment }}' != macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
           fi
-          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          elif [[ '${{ matrix.environment }}' == docker ]]; then
+          fi
+          if [[ '${{ matrix.environment }}' == docker ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
           elif [[ '${{ matrix.environment }}' == macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")

AGENTS.md

@@ -15,6 +15,13 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev
 - Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
 - Read `README.md` for the full setup, CLI, Docker, API, and release surface.
 
+## Concurrency Contract
+
+- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
+- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
+- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
+- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
+
 ## Development Setup
 
 ```bash

Dockerfile

@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.227 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.256 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.227
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.256
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base

archivebox/api/v1_core.py

@@ -11,7 +11,7 @@ from uuid import UUID
 from typing import Union, Any, Annotated
 from datetime import datetime, time
 
-from django.db import transaction
+from django.db import IntegrityError
 from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
@@ -517,52 +517,53 @@ def create_archiveresult(
     normalized_status = ArchiveResult.normalize_status(status)
     parsed_output_json = _parse_archiveresult_output_json(output_json)
     hook = hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME
-    existing_result = ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin_name).first()
-    existing_output_files = dict(existing_result.output_files or {}) if existing_result else {}
-    output_files = _write_archiveresult_files(
+    result_lookup = {
+        "snapshot": snapshot,
+        "plugin": plugin_name,
+        "hook_name": hook,
+    }
+    uploaded_output_files = _write_archiveresult_files(
         request,
         snapshot,
         plugin_name,
-        existing_output_files=existing_output_files,
         allow_empty=True,
     )
-    now = timezone.now()
-
-    with transaction.atomic():
-        Snapshot.objects.select_for_update().get(pk=snapshot.pk)
-        existing_result = ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin_name).first()
-        if existing_result:
-            output_files = {
-                **dict(existing_result.output_files or {}),
-                **output_files,
-            }
-            result = existing_result
-        else:
-            existing_result = None
-            result = ArchiveResult(
-                snapshot=snapshot,
-                plugin=plugin_name,
-                hook_name=hook,
-            )
-
-        if (
-            existing_result
-            and normalized_status == ArchiveResult.StatusChoices.STARTED
-            and existing_result.status != ArchiveResult.StatusChoices.STARTED
-        ):
-            normalized_status = existing_result.status
+    result = ArchiveResult.objects.filter(**result_lookup).first()
+    for _attempt in range(3):
+        output_files = {
+            **(result.output_file_map() if result else {}),
+            **uploaded_output_files,
+        }
         output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
         output_file_paths = list(output_files.keys())
-        result.status = normalized_status
-        result.hook_name = hook
-        result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
-        result.output_json = parsed_output_json
-        result.output_files = output_files
-        result.output_size = output_size
-        result.output_mimetypes = output_mimetypes
-        result.start_ts = result.start_ts or now
-        result.end_ts = now
-        result.save()
+        result_status = normalized_status
+        if result and result_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+            result_status = result.status
+        now = timezone.now()
+        values = {
+            "status": result_status,
+            "output_str": output_str or (output_file_paths[0] if output_file_paths else ""),
+            "output_json": parsed_output_json,
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "start_ts": result.start_ts or now if result else now,
+            "end_ts": now,
+        }
+        if result:
+            if result.safe_update(values):
+                break
+            continue
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_name,
+            hook,
+            defaults=values,
+        )
+        if created:
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
 
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(snapshot)
@@ -576,44 +577,46 @@ def patch_archiveresult(
 ):
     """Append or replace files on an existing ArchiveResult."""
     result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    output_files = _write_archiveresult_files(
+    uploaded_output_files = _write_archiveresult_files(
         request,
         result.snapshot,
         result.plugin,
-        existing_output_files=result.output_file_map(),
     )
-    latest_result = ArchiveResult.objects.only("output_files", "status").get(pk=result.pk)
-    output_files = {
-        **latest_result.output_file_map(),
-        **output_files,
-    }
-    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-
-    update_fields = ["output_files", "output_size", "output_mimetypes", "end_ts", "modified_at"]
-    result.output_files = output_files
-    result.output_size = output_size
-    result.output_mimetypes = output_mimetypes
-    result.end_ts = timezone.now()
     output_str = _get_archiveresult_upload_form_value(request, "output_str")
     status = _get_archiveresult_upload_form_value(request, "status")
     output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    if output_str:
-        result.output_str = output_str
-        update_fields.append("output_str")
-    if status:
-        normalized_status = ArchiveResult.normalize_status(status)
-        if normalized_status == ArchiveResult.StatusChoices.STARTED and latest_result.status != ArchiveResult.StatusChoices.STARTED:
-            normalized_status = latest_result.status
-        result.status = normalized_status
-        update_fields.append("status")
-    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
-        result.status = ArchiveResult.StatusChoices.SUCCEEDED
-        update_fields.append("status")
-    if output_json:
-        result.output_json = _parse_archiveresult_output_json(output_json)
-        update_fields.append("output_json")
+    parsed_output_json = _parse_archiveresult_output_json(output_json) if output_json else None
+
+    if not ArchiveResult.output_files_upload_complete(uploaded_output_files):
+        result.output_files = {**result.output_file_map(), **uploaded_output_files}
+        result.output_size, result.output_mimetypes = _summarize_archiveresult_output_files(result.output_files)
+        return result
+
+    for _attempt in range(3):
+        output_files = {**result.output_file_map(), **uploaded_output_files}
+        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+        values: dict[str, Any] = {
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "end_ts": timezone.now(),
+        }
+        if output_str:
+            values["output_str"] = output_str
+        if status:
+            normalized_status = ArchiveResult.normalize_status(status)
+            if normalized_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+                normalized_status = result.status
+            values["status"] = normalized_status
+        elif result.status == ArchiveResult.StatusChoices.QUEUED:
+            values["status"] = ArchiveResult.StatusChoices.SUCCEEDED
+        if output_json:
+            values["output_json"] = parsed_output_json
+        if result.safe_update(values):
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
 
-    result.save(update_fields=update_fields)
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(result.snapshot)
 
@@ -930,52 +933,47 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    # Browser clients sync metadata and upload capture artifacts immediately
-    # after queueing a URL. The runner holds the crawl lifecycle lock for the
-    # whole crawl, so idempotently posting an already-created snapshot must not
-    # wait behind that potentially long-running archive job.
-    existing_snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if existing_snapshot is not None and (status is None or existing_snapshot.status == status):
-        if data.title is not None and existing_snapshot.title != data.title:
-            Snapshot.objects.filter(pk=existing_snapshot.pk).update(title=data.title, modified_at=timezone.now())
-            existing_snapshot.title = data.title
-        if tags:
-            existing_snapshot.save_tags(tags)
-        setattr(request, "with_archiveresults", False)
-        return existing_snapshot
+    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
+    # lock. The unique insert recovery and CAS update below are the request-side
+    # coordination boundary for this idempotent metadata sync.
+    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None:
+        try:
+            snapshot = Snapshot.objects.create(
+                url=data.url,
+                crawl=crawl,
+                depth=data.depth,
+                title=data.title,
+                timestamp=str(timezone.now().timestamp()),
+                status=status or Snapshot.StatusChoices.QUEUED,
+                retry_at=timezone.now(),
+            )
+        except IntegrityError:
+            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+            if snapshot is None:
+                raise
 
-    with crawl_lifecycle_lock(str(crawl.id)):
-        snapshot_defaults = {
-            "depth": data.depth,
-            "title": data.title,
-            "timestamp": str(timezone.now().timestamp()),
-            "status": status or Snapshot.StatusChoices.QUEUED,
-            "retry_at": timezone.now(),
-        }
-        snapshot, _ = Snapshot.objects.get_or_create(
-            url=data.url,
-            crawl=crawl,
-            defaults=snapshot_defaults,
+    for _attempt in range(3):
+        updates: dict[str, Any] = {}
+        if data.title is not None and snapshot.title != data.title:
+            updates["title"] = data.title
+        if status is not None and snapshot.status != status:
+            updates["status"] = status
+        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
+            break
+    else:
+        raise HttpError(409, "Snapshot changed while metadata was being updated")
+
+    if tags:
+        snapshot.save_tags(
+            tags,
+            created_by=request.user if isinstance(request.user, User) else None,
         )
 
-        update_fields: list[str] = []
-        if data.title is not None and snapshot.title != data.title:
-            snapshot.title = data.title
-            update_fields.append("title")
-        if status is not None and snapshot.status != status:
-            snapshot.status = status
-            update_fields.append("status")
-        if update_fields:
-            update_fields.append("modified_at")
-            snapshot.save(update_fields=update_fields)
-
-        if tags:
-            snapshot.save_tags(tags)
-
-        try:
-            snapshot.ensure_crawl_symlink()
-        except Exception:
-            pass
+    try:
+        snapshot.ensure_crawl_symlink()
+    except Exception:
+        pass
 
     setattr(request, "with_archiveresults", False)
     return snapshot
@@ -1023,7 +1021,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
             update_fields.append("retry_at")
 
         if tags is not None:
-            snapshot.save_tags(normalize_tag_list(tags))
+            snapshot.save_tags(
+                normalize_tag_list(tags),
+                created_by=request.user if isinstance(request.user, User) else None,
+            )
 
         if payload.get("status") == Snapshot.StatusChoices.SEALED:
             snapshot.cancel()
@@ -1410,8 +1411,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    # Add the tag to the snapshot
-    snapshot.tags.add(tag.pk)
+    snapshot.add_tag_ids([tag.pk])
 
     return {
         "success": True,
@@ -1439,8 +1439,7 @@ def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSche
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    # Remove the tag from the snapshot
-    snapshot.tags.remove(tag.pk)
+    snapshot.remove_tag_ids([tag.pk])
 
     return {
         "success": True,

archivebox/cli/archivebox_add.py

@@ -421,13 +421,13 @@ def main(**kwargs):
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
         if extract:
-            from abx_dl.models import discover_plugins, plugins_matching_output
+            from archivebox.plugins.discovery import get_plugin_catalog
 
-            all_plugins = discover_plugins()
+            catalog = get_plugin_catalog()
             tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in all_plugins}
+            plugin_names = {name.lower(): name for name in catalog}
             selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += plugins_matching_output(all_plugins, tokens)
+            selected += catalog.matching_output(tokens)
             if not selected:
                 raise click.UsageError(f"No plugins found matching extract types: {extract}")
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]

archivebox/cli/archivebox_archiveresult.py

@@ -67,7 +67,7 @@ def create_archiveresults(
     Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
     Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
     If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits one request for each enabled Snapshot plugin.
+    Otherwise, emits requests for all enabled snapshot hooks.
 
     Exit codes:
         0: Success
@@ -144,17 +144,18 @@ def create_archiveresults(
 
     created_count = 0
     for snapshot in snapshots:
-        if plugin:
+        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+        hooks = [
+            hook
+            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
+            if not plugin or hook.parent.name == plugin
+        ]
+        for hook_path in hooks:
+            hook_name = hook_path.stem
+            plugin_name = hook_path.parent.name
             if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
+                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
             created_count += 1
-        else:
-            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-            hooks = discover_hooks("Snapshot", filter_disabled=True, config=config)
-            for plugin_name in dict.fromkeys(hook_path.parent.name for hook_path in hooks):
-                if not is_tty:
-                    write_record(build_archiveresult_request(snapshot.id, plugin_name, status=status))
-                created_count += 1
 
     rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
     return 0
@@ -254,11 +255,9 @@ def update_archiveresults(
 
             # Apply updates from CLI flags
             if status:
-                if status == ArchiveResult.StatusChoices.QUEUED:
-                    result.reset_for_retry()
-                else:
-                    result.status = status
-                    result.save(update_fields=["status", "modified_at"])
+                result.status = status
+
+            result.save()
             updated_count += 1
 
             if not is_tty:

archivebox/cli/archivebox_extract.py

@@ -70,7 +70,12 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
             )
             return 1
 
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        run_crawl(
+            str(snapshot.crawl_id),
+            snapshot_ids=[str(snapshot.id)],
+            selected_plugins=[archiveresult.plugin],
+            selected_plugins_are_explicit=False,
+        )
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -122,6 +127,7 @@ def run_plugins(
     from archivebox.core.models import Snapshot
     from archivebox.core.models import ArchiveResult
     from archivebox.services.runner import run_crawl
+    from archivebox.plugins.discovery import get_plugin_catalog
 
     is_tty = sys.stdout.isatty()
 
@@ -142,6 +148,8 @@ def run_plugins(
     # Gather snapshot IDs and optional plugin constraints to process
     snapshot_ids = set()
     requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
+    requested_hooks_by_snapshot: dict[str, set[tuple[str, str]]] = defaultdict(set)
+    plugin_level_requests_by_snapshot: dict[str, set[str]] = defaultdict(set)
     for record in records:
         record_type = record.get("type")
 
@@ -163,7 +171,14 @@ def run_plugins(
                 snapshot_ids.add(str(snapshot_id))
                 plugin_name = record.get("plugin")
                 if plugin_name and not plugins_list:
-                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
+                    snapshot_key = str(snapshot_id)
+                    plugin_key = str(plugin_name)
+                    requested_plugins_by_snapshot[snapshot_key].add(plugin_key)
+                    hook_name = str(record.get("hook_name") or "")
+                    if hook_name:
+                        requested_hooks_by_snapshot[snapshot_key].add((plugin_key, hook_name))
+                    else:
+                        plugin_level_requests_by_snapshot[snapshot_key].add(plugin_key)
 
         elif "id" in record:
             # Assume it's a snapshot ID
@@ -192,21 +207,38 @@ def run_plugins(
             if snapshot_id in existing_snapshot_ids
             for plugin_name in plugin_names
         )
-    queued_rows: set[tuple[str, str]] = set()
-    if preserve_queued and requested_pairs:
-        queued_rows = {
-            (str(snapshot_id), plugin_name)
-            for snapshot_id, plugin_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name in requested_pairs},
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin")
+    plugins_by_name = get_plugin_catalog()
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        exact_hook_names = {
+            hook_name
+            for requested_plugin, hook_name in requested_hooks_by_snapshot.get(snapshot_id, set())
+            if requested_plugin == plugin_name
         }
-    rows_to_queue = requested_pairs - queued_rows
+        if exact_hook_names and plugin_name not in plugin_level_requests_by_snapshot.get(snapshot_id, set()):
+            requested_rows.update((snapshot_id, plugin_name, hook_name) for hook_name in exact_hook_names)
+            continue
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))
+
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
 
     reset_fields = {
         "status": ArchiveResult.StatusChoices.QUEUED,
-        "hook_name": "",
         "output_str": "",
         "output_json": None,
         "output_files": {},
@@ -217,28 +249,28 @@ def run_plugins(
         "modified_at": timezone.now(),
     }
     if rows_to_queue and plugins_list:
-        rows_to_reset_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name in rows_to_queue:
-            rows_to_reset_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in rows_to_reset_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
                 **reset_fields,
             )
     elif rows_to_queue and requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name in rows_to_queue:
-            snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
                 **reset_fields,
             )
     existing_rows = (
         {
-            (str(snapshot_id), plugin_name)
-            for snapshot_id, plugin_name in ArchiveResult.objects.filter(
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
                 snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name in rows_to_queue},
-            ).values_list("snapshot_id", "plugin")
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
         }
         if rows_to_queue
         else set()
@@ -250,10 +282,10 @@ def run_plugins(
                 ArchiveResult(
                     snapshot_id=snapshot_id,
                     plugin=plugin_name,
-                    hook_name="",
+                    hook_name=hook_name,
                     status=ArchiveResult.StatusChoices.QUEUED,
                 )
-                for snapshot_id, plugin_name in sorted(missing_rows)
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
             ],
             batch_size=500,
         )
@@ -261,13 +293,12 @@ def run_plugins(
     processed_count = len(existing_snapshot_ids)
     queue_at = timezone.now()
     if existing_snapshot_ids:
-        if requested_pairs:
-            # Search indexing on a sealed Snapshot is the only targeted plugin
-            # allowed to bypass the normal lifecycle. Every other requested
-            # plugin requeues its Snapshot through the unified lifecycle.
-            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name in rows_to_queue}
+        if requested_rows:
+            # Explicit plugin retries are maintenance on the existing snapshot;
+            # preserve a sealed lifecycle while making its queued rows due.
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
             if preserve_queued and queued_rows:
-                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name in queued_rows}
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
                 affected_snapshot_ids.update(
                     str(snapshot_id)
                     for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
@@ -282,18 +313,12 @@ def run_plugins(
                     )
                 )
             requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
-            for snapshot_id, plugin_name in requested_pairs:
+            for snapshot_id, plugin_name, _hook_name in requested_rows:
                 requested_plugins_by_id[snapshot_id].add(plugin_name)
             for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
-                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
-                sealed_search_backfill = (
-                    snapshot.status == Snapshot.StatusChoices.SEALED
-                    and plugin_names
-                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
-                )
-                if sealed_search_backfill:
+                if snapshot.status == Snapshot.StatusChoices.SEALED and requested_plugins_by_id.get(str(snapshot.id)):
                     snapshot.safe_update(
                         {"retry_at": queue_at, "modified_at": queue_at},
                         refresh=False,
@@ -318,7 +343,7 @@ def run_plugins(
                     refresh=False,
                     extra_filter={"status": snapshot.status},
                 )
-    if existing_crawl_ids and not requested_pairs:
+    if existing_crawl_ids and not requested_rows:
         from archivebox.crawls.models import Crawl
 
         for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
@@ -362,7 +387,7 @@ def run_plugins(
             selected_plugins = (
                 plugins_list
                 or sorted(
-                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
+                    {plugin for snapshot_id, plugin, _hook_name in requested_rows if snapshot_id in crawl_snapshot_ids},
                 )
                 or None
             )
@@ -371,6 +396,7 @@ def run_plugins(
                 snapshot_ids=sorted(crawl_snapshot_ids),
                 selected_plugins=selected_plugins,
                 show_progress=show_progress,
+                selected_plugins_are_explicit=bool(plugins_list),
             )
 
     if not emit_results:

archivebox/cli/archivebox_install.py

@@ -17,12 +17,12 @@ def _resolve_install_targets(
     requested_names: tuple[str, ...],
 ) -> tuple[list[str], list[str]]:
     """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from abx_dl.models import discover_plugins, filter_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
-    plugins = discover_plugins(runtime="archivebox")
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in plugins}
+    catalog = get_plugin_catalog()
+    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
     plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in plugins.items():
+    for plugin_name, plugin in catalog.items():
         for required_binary in plugin.config.required_binaries:
             aliases = {required_binary.name.lower()}
             template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
@@ -47,7 +47,7 @@ def _resolve_install_targets(
         else:
             raw_binary_names.append(name)
 
-    selected_plugins = filter_plugins(plugins, requested_plugins, include_providers=True) if requested_plugins else {}
+    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
     selected_plugin_names = {name.lower() for name in selected_plugins}
     raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
     return sorted(selected_plugins), sorted(set(raw_binary_names))

archivebox/cli/archivebox_run.py

@@ -156,18 +156,7 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_ARCHIVERESULT:
-                if record_id:
-                    # Existing archiveresult - re-queue
-                    try:
-                        archiveresult = ArchiveResult.objects.get(id=record_id)
-                    except ArchiveResult.DoesNotExist:
-                        archiveresult = None
-                else:
-                    archiveresult = None
-
-                snapshot_id = record.get("snapshot_id")
-                plugin_name = record.get("plugin")
-                snapshot = None
+                archiveresult = ArchiveResult.from_json(record)
                 if archiveresult:
                     if archiveresult.status in [
                         ArchiveResult.StatusChoices.FAILED,
@@ -177,12 +166,10 @@ def process_stdin_records() -> int:
                     ]:
                         archiveresult.reset_for_retry()
                     snapshot = archiveresult.snapshot
-                    plugin_name = plugin_name or archiveresult.plugin
-                elif snapshot_id:
-                    try:
-                        snapshot = Snapshot.objects.get(id=snapshot_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = None
+                    plugin_name = archiveresult.plugin
+                else:
+                    snapshot = None
+                    plugin_name = None
 
                 if snapshot:
                     snapshot.queue_for_extraction()

archivebox/cli/archivebox_update.py

@@ -77,7 +77,7 @@ def reindex_snapshots(
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
     from archivebox.core.models import ArchiveResult, Snapshot
-    from abx_dl.models import discover_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
     # Search backfill is the one maintenance hook allowed to execute without
     # reopening a Snapshot. Restrict that exception to already-sealed rows;
@@ -86,7 +86,7 @@ def reindex_snapshots(
 
     stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
-    plugins_by_name = discover_plugins(runtime="archivebox")
+    plugins_by_name = get_plugin_catalog()
     required_hooks_by_plugin = {
         plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
         for plugin_name in search_plugins

archivebox/cli/archivebox_version.py

@@ -265,15 +265,14 @@ def version(
     seen_failures: set[str] = set()
     seen_rows: set[tuple[str, str, str, str]] = set()
 
-    from archivebox.plugins.discovery import get_enabled_plugins
+    from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from abx_dl.config import get_required_binary_requests
     from abx_dl.dependencies import resolve_binary_requests
-    from abx_dl.models import discover_plugins, filter_plugins
     from abx_dl.orchestrator import create_bus
     from abxpkg.binary_service import BinaryEvent, BinaryService
 
-    plugins = discover_plugins(runtime="archivebox")
-    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    plugins = get_plugin_catalog()
+    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
     enabled_plugin_names = set(enabled_plugins)
     runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
     derived_config: dict[str, object] = {}

archivebox/config/collection.py

@@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
     so they round-trip through INI's string-only storage. When reading the
     file back into ``Machine.config`` (a JSONField that holds native types)
     those strings have to be decoded — otherwise downstream consumers like
-    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    ``ExecutionPlan.seed_config`` → ``MachineEvent`` → abx-dl see a JSON string
     where they expect a dict and raise ``TypeError``.
     Declared fields go through pydantic-settings' own ``field_is_complex`` /
     ``prepare_field_value`` so they're decoded per annotation. Undeclared

archivebox/config/common.py

@@ -852,13 +852,11 @@ class ArchiveBoxBaseConfig(
             disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
         selected_plugin_roots = plugin_names
         if selected_plugin_roots:
-            from abx_dl.models import discover_plugins, filter_plugins
+            from archivebox.plugins.discovery import get_plugin_catalog
 
             selected_plugins = set(
-                filter_plugins(
-                    discover_plugins(runtime="archivebox"),
+                get_plugin_catalog().select(
                     sorted(selected_plugin_roots),
-                    include_providers=True,
                     disabled_names=disabled_plugins,
                 ),
             )

archivebox/core/admin_snapshots.py

@@ -346,19 +346,8 @@ class SnapshotAdminForm(forms.ModelForm):
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
-            if tags_str:
-                tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
-                tags = []
-                for name in tag_names:
-                    tag, _ = Tag.objects.get_or_create(
-                        name__iexact=name,
-                        defaults={"name": name},
-                    )
-                    tag = Tag.objects.filter(name__iexact=name).first() or tag
-                    tags.append(tag)
-                instance.tags.set(tags)
-            else:
-                instance.tags.clear()
+            tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+            instance.save_tags(tag_names)
 
         return instance
 
@@ -367,6 +356,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     form = SnapshotAdminForm
     raw_id_fields = ("crawl", "parent_snapshot")
     list_select_related = ()
+
+    def save_related(self, request, form, formsets, change):
+        super().save_related(request, form, formsets, change)
+        tags_str = form.cleaned_data.get("tags_editor", "")
+        tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+        form.instance.save_tags(
+            tag_names,
+            created_by=request.user if request.user.is_authenticated else None,
+        )
+
     list_display = (
         "permissions_badge",
         "created_at",

archivebox/core/migrations/0052_unique_archiveresult_per_snapshot_plugin.py

@@ -1,71 +0,0 @@
-from django.db import migrations, models
-from django.db.models import Sum
-
-
-def consolidate_archiveresults_per_plugin(apps, schema_editor):
-    ArchiveResult = apps.get_model("core", "ArchiveResult")
-    Snapshot = apps.get_model("core", "Snapshot")
-    duplicate_groups = ArchiveResult.objects.values("snapshot_id", "plugin").annotate(count=models.Count("id")).filter(count__gt=1)
-
-    affected_snapshot_ids = set()
-    for group in duplicate_groups.iterator(chunk_size=200):
-        rows = list(
-            ArchiveResult.objects.filter(
-                snapshot_id=group["snapshot_id"],
-                plugin=group["plugin"],
-            ).order_by("created_at", "id"),
-        )
-        winner = max(
-            rows,
-            key=lambda row: (
-                bool(row.output_files),
-                int(row.output_size or 0),
-                row.modified_at,
-                str(row.id),
-            ),
-        )
-        output_files = {}
-        mimetypes = set()
-        for row in rows:
-            output_files.update(row.output_files or {})
-            mimetypes.update(part.strip() for part in (row.output_mimetypes or "").split(",") if part.strip())
-
-        winner.output_files = output_files
-        winner.output_size = max(
-            sum(int(metadata.get("size") or 0) for metadata in output_files.values() if isinstance(metadata, dict)),
-            *(int(row.output_size or 0) for row in rows),
-        )
-        winner.output_mimetypes = ",".join(sorted(mimetypes))
-        winner.start_ts = min((row.start_ts for row in rows if row.start_ts), default=None)
-        winner.end_ts = max((row.end_ts for row in rows if row.end_ts), default=None)
-        winner.save(update_fields=["output_files", "output_size", "output_mimetypes", "start_ts", "end_ts"])
-        ArchiveResult.objects.filter(id__in=[row.id for row in rows if row.id != winner.id]).delete()
-        affected_snapshot_ids.add(group["snapshot_id"])
-
-    for snapshot_id in affected_snapshot_ids:
-        total = ArchiveResult.objects.filter(snapshot_id=snapshot_id).aggregate(total=Sum("output_size"))["total"] or 0
-        Snapshot.objects.filter(id=snapshot_id).update(output_size=total)
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("core", "0051_postgres_url_pattern_ops_index"),
-    ]
-
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="archiveresult",
-            name="unique_archiveresult_per_snapshot_hook",
-        ),
-        migrations.RunPython(
-            consolidate_archiveresults_per_plugin,
-            reverse_code=migrations.RunPython.noop,
-        ),
-        migrations.AddConstraint(
-            model_name="archiveresult",
-            constraint=models.UniqueConstraint(
-                fields=("snapshot", "plugin"),
-                name="unique_archiveresult_per_snapshot_plugin",
-            ),
-        ),
-    ]

archivebox/core/migrations/0053_alter_archiveresult_options.py

@@ -5,7 +5,7 @@ from django.db import migrations
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("core", "0052_unique_archiveresult_per_snapshot_plugin"),
+        ("core", "0051_postgres_url_pattern_ops_index"),
     ]
 
     operations = [

archivebox/core/models.py

@@ -12,7 +12,7 @@ from urllib.parse import urlparse
 from django.conf import settings
 from django.contrib import admin
 from django.core.exceptions import FieldDoesNotExist, ObjectDoesNotExist, ValidationError
-from django.db import models, transaction
+from django.db import IntegrityError, models, transaction
 from django.db.models import Case, F, Q, QuerySet, Sum, Value, When
 from django.db.models.fields.json import KT
 from django.db.models.functions import Coalesce, Concat
@@ -85,6 +85,19 @@ class Tag(ModelWithUUID):
     modified_at = models.DateTimeField(auto_now=True)
     name = models.CharField(unique=True, blank=False, max_length=100)
 
+    @classmethod
+    def get_or_create_by_name(cls, name: str, *, defaults: Mapping[str, Any] | None = None) -> tuple["Tag", bool]:
+        tag = cls.objects.filter(name__iexact=name).first()
+        if tag:
+            return tag, False
+        try:
+            return cls.objects.create(name=name, **(defaults or {})), True
+        except IntegrityError:
+            tag = cls.objects.filter(name__iexact=name).first()
+            if tag is None:
+                raise
+            return tag, False
+
     snapshot_set: models.Manager["Snapshot"]
 
     class Meta(ModelWithUUID.Meta):
@@ -139,7 +152,7 @@ class Tag(ModelWithUUID):
         if not name:
             return None
 
-        tag, _ = Tag.objects.get_or_create(name=name)
+        tag, _ = Tag.get_or_create_by_name(name)
 
         # Auto-attach to snapshot if in overrides
         if overrides and "snapshot" in overrides and tag:
@@ -606,13 +619,25 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     archiveresult_set: models.Manager["ArchiveResult"]
 
     def add_tag_ids(self, tag_ids: Iterable[int | str]) -> None:
+        tag_ids = [tag_id for tag_id in dict.fromkeys(tag_ids) if tag_id]
+        for tag_id in tag_ids:
+            try:
+                SnapshotTag(snapshot_id=self.pk, tag_id=tag_id).save(force_insert=True)
+            except IntegrityError:
+                # Only the unique (snapshot, tag) conflict is idempotent. Do
+                # not hide foreign-key or other integrity failures.
+                if SnapshotTag.objects.filter(snapshot_id=self.pk, tag_id=tag_id).exists():
+                    continue
+                raise
+
+    def remove_tag_ids(self, tag_ids: Iterable[int | str]) -> int:
         tag_ids = [tag_id for tag_id in dict.fromkeys(tag_ids) if tag_id]
         if not tag_ids:
-            return
-        SnapshotTag.objects.bulk_create(
-            [SnapshotTag(snapshot_id=self.pk, tag_id=tag_id) for tag_id in tag_ids],
-            ignore_conflicts=True,
-        )
+            return 0
+        # QuerySet.delete() wraps even a fast through-table DELETE in atomic().
+        # SnapshotTag has no delete hooks or child rows, so issue the same
+        # idempotent DELETE as one autocommit statement.
+        return SnapshotTag.objects.filter(snapshot_id=self.pk, tag_id__in=tag_ids)._raw_delete(SnapshotTag.objects.db)
 
     class Meta(
         ModelWithDeleteAfter.Meta,
@@ -972,7 +997,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     @property
     def binary_set(self):
         """Get all Binary objects used by processes related to this snapshot."""
-        from archivebox.machine.models import Binary
 
         return Binary.objects.filter(process_set__archiveresult__snapshot_id=self.id).distinct()
 
@@ -1786,8 +1810,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     def _merge_tags_from_index(self, index_data: dict):
         """Merge tags - union of both sources."""
-        from django.db import transaction
-
         index_tags = set(index_data.get("tags", "").split(",")) if index_data.get("tags") else set()
         index_tags = {t.strip() for t in index_tags if t.strip()}
 
@@ -1795,14 +1817,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         new_tags = index_tags - db_tags
         if new_tags:
-            with transaction.atomic():
-                for tag_name in new_tags:
-                    tag, _ = Tag.objects.get_or_create(name=tag_name)
-                    self.add_tag_ids([tag.pk])
+            for tag_name in new_tags:
+                tag, _ = Tag.get_or_create_by_name(tag_name)
+                self.add_tag_ids([tag.pk])
 
     def _merge_archive_results_from_index(self, index_data: dict, update_existing: bool = True):
-        """Merge ArchiveResults one row per plugin; retries update the existing row."""
-        existing = {ar.plugin: ar for ar in ArchiveResult.objects.filter(snapshot=self)}
+        """Merge ArchiveResults one row per hook; retries update the existing row."""
+        existing = {(ar.plugin, ar.hook_name): ar for ar in ArchiveResult.objects.filter(snapshot=self)}
         if update_existing:
             for archiveresult in existing.values():
                 normalized_status = ArchiveResult.normalize_status(archiveresult.status)
@@ -1865,7 +1886,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         output_mimetypes = result_data.get("output_mimetypes", "")
 
         hook_name = result_data.get("hook_name", "")
-        existing_result = existing.get(plugin)
+        existing_result = existing.get((plugin, hook_name))
         if existing_result:
             if not update_existing:
                 return
@@ -1874,9 +1895,6 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             if existing_result.status != status:
                 existing_result.status = status
                 update_fields.append("status")
-            if hook_name and existing_result.hook_name != hook_name:
-                existing_result.hook_name = hook_name
-                update_fields.append("hook_name")
             if output_str and existing_result.output_str != output_str:
                 existing_result.output_str = output_str
                 update_fields.append("output_str")
@@ -1933,7 +1951,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 end_ts=end_ts,
                 process=process,
             )
-        existing[plugin] = archiveresult
+        existing[(plugin, hook_name)] = archiveresult
 
     def write_index_json(self):
         """Write index.json in 0.9.x format (deprecated, use write_index_jsonl)."""
@@ -2263,15 +2281,18 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             for tag in dup.tags.all():
                 keeper.add_tag_ids([tag.pk])
 
-            # Move ArchiveResults while preserving the one-row-per-plugin invariant.
+            # Move each hook result, merging only an exact identity collision.
             for result in ArchiveResult.objects.filter(snapshot=dup):
-                existing = ArchiveResult.objects.filter(snapshot=keeper, plugin=result.plugin).first()
+                existing = ArchiveResult.objects.filter(
+                    snapshot=keeper,
+                    plugin=result.plugin,
+                    hook_name=result.hook_name,
+                ).first()
                 if existing is None:
                     result.snapshot = keeper
                     result.save(update_fields=["snapshot", "modified_at"])
                     continue
-                prefer_result = result.output_size > existing.output_size
-                prior_output_sizes = (existing.output_size, result.output_size)
+
                 output_files = {**(existing.output_files or {}), **(result.output_files or {})}
                 existing.output_files = output_files
                 existing.output_size = max(
@@ -2280,13 +2301,15 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                         for metadata in output_files.values()
                         if isinstance(metadata, dict)
                     ),
-                    *prior_output_sizes,
+                    existing.output_size,
+                    result.output_size,
                 )
-                if prefer_result:
+                if result.modified_at >= existing.modified_at:
                     existing.status = result.status
-                    existing.hook_name = result.hook_name
                     existing.output_str = result.output_str
                     existing.output_json = result.output_json
+                    existing.start_ts = result.start_ts
+                    existing.end_ts = result.end_ts
                 existing.output_mimetypes = ",".join(
                     sorted(
                         {
@@ -2765,10 +2788,13 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     def archive_size(self):
         return int(self.output_size or 0)
 
-    def save_tags(self, tags: Iterable[str] = ()) -> None:
-        tags_id = [Tag.objects.get_or_create(name=tag)[0].pk for tag in tags if tag.strip()]
-        self.tags.clear()
-        self.add_tag_ids(tags_id)
+    def save_tags(self, tags: Iterable[str] = (), *, created_by: Any = None) -> None:
+        from archivebox.core.tag_util import get_or_create_tag
+
+        tag_ids = {get_or_create_tag(tag, created_by=created_by)[0].pk for tag in tags if tag.strip()}
+        existing_tag_ids = set(SnapshotTag.objects.filter(snapshot_id=self.pk).values_list("tag_id", flat=True))
+        self.remove_tag_ids(existing_tag_ids - tag_ids)
+        self.add_tag_ids(tag_ids - existing_tag_ids)
 
     def pending_archiveresults(self) -> QuerySet["ArchiveResult"]:
         return self.archiveresult_set.exclude(status__in=ArchiveResult.FINAL_OR_ACTIVE_STATES)
@@ -3036,8 +3062,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         - abx_plugins/plugins/*/on_Snapshot__*.{py,sh,js}
         - data/custom_plugins/*/on_Snapshot__*.{py,sh,js}
 
-        Creates one ArchiveResult per plugin. A queued plugin row runs all of that
-        plugin's Snapshot hooks; hook_name records the most recently completed hook.
+        Creates one ArchiveResult per hook (not per plugin), with hook_name set.
+        This enables step-based execution where all hooks in a step can run in parallel.
         """
         try:
             self.validate_url_for_archiving()
@@ -3055,21 +3081,17 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             hooks = ((hook_path.parent.name, hook_path.stem) for hook_path in discover_hooks("Snapshot", config=config))
         archiveresults = []
 
-        for plugin in dict.fromkeys(plugin for plugin, _hook_name in hooks):
-            # ArchiveResult output is one filesystem directory per plugin, so all
-            # hooks and retries update this row in place instead of creating siblings.
-            archiveresult, _created = ArchiveResult.objects.get_or_create(
-                snapshot=self,
-                plugin=plugin,
+        for plugin, hook_name in hooks:
+            # Hooks in one plugin share a filesystem directory, but each hook has
+            # its own durable result row and retries update that exact row.
+            archiveresult, _created = ArchiveResult.get_or_create_by_hook(
+                self,
+                plugin,
+                hook_name,
                 defaults={
-                    "hook_name": "",
                     "status": ArchiveResult.INITIAL_STATE,
                 },
             )
-            if archiveresult.hook_name == self.BROWSER_EXTENSION_UPLOAD_HOOK_NAME:
-                archiveresult.hook_name = ""
-                archiveresult.status = ArchiveResult.INITIAL_STATE
-                archiveresult.save(update_fields=["hook_name", "status", "modified_at"])
             if archiveresult.status == ArchiveResult.INITIAL_STATE:
                 archiveresults.append(archiveresult)
 
@@ -3158,10 +3180,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             snapshot=self,
             status=ArchiveResult.StatusChoices.FAILED,
         )
+        legacy_result_count = retryable_results.filter(hook_name="").count()
         now = timezone.now()
-        count = retryable_results.update(
+        count = retryable_results.exclude(hook_name="").update(
             status=ArchiveResult.StatusChoices.QUEUED,
-            hook_name="",
             output_str="",
             output_json=None,
             output_files={},
@@ -3172,11 +3194,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             modified_at=now,
         )
 
-        if count > 0:
+        if count + legacy_result_count > 0:
             self.refresh_from_db(fields=["modified_at", "retry_at", "status"])
             self.queue_for_extraction(when=now)
 
-        return count
+        return count + legacy_result_count
 
     # =========================================================================
     # URL Helper Properties (migrated from Link schema)
@@ -3850,6 +3872,27 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             "backoff": cls.StatusChoices.BACKOFF,
         }.get(str(status or "").strip().lower(), cls.StatusChoices.FAILED)
 
+    @classmethod
+    def get_or_create_by_hook(
+        cls,
+        snapshot: Snapshot,
+        plugin: str,
+        hook_name: str,
+        *,
+        defaults: Mapping[str, Any] | None = None,
+    ) -> tuple["ArchiveResult", bool]:
+        lookup = {"snapshot": snapshot, "plugin": plugin, "hook_name": hook_name}
+        result = cls.objects.filter(**lookup).first()
+        if result:
+            return result, False
+        try:
+            return cls.objects.create(**lookup, **(defaults or {})), True
+        except IntegrityError:
+            result = cls.objects.filter(**lookup).first()
+            if result is None:
+                raise
+            return result, False
+
     @staticmethod
     def output_files_upload_complete(output_files: dict[str, dict[str, Any]]) -> bool:
         if not output_files:
@@ -3984,7 +4027,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             models.Index(fields=["-start_ts", "-id"], name="archiveresult_start_idx"),
         ]
         constraints: ClassVar[list[models.BaseConstraint]] = [
-            models.UniqueConstraint(fields=["snapshot", "plugin"], name="unique_archiveresult_per_snapshot_plugin"),
+            models.UniqueConstraint(fields=["snapshot", "plugin", "hook_name"], name="unique_archiveresult_per_snapshot_hook"),
         ]
 
     def __str__(self):
@@ -4095,16 +4138,16 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             except ArchiveResult.DoesNotExist:
                 pass
 
-        # Get or create by snapshot_id + plugin. The filesystem has one output
-        # directory per plugin, so every hook and retry updates that same DB row.
+        # Get or create by the durable scheduler identity. Hooks in one plugin
+        # share an output directory, while retries update only their exact row.
         try:
             snapshot = Snapshot.objects.get(id=snapshot_id)
 
-            result, _ = ArchiveResult.objects.get_or_create(
-                snapshot=snapshot,
-                plugin=plugin,
+            result, _ = ArchiveResult.get_or_create_by_hook(
+                snapshot,
+                plugin,
+                record.get("hook_name", ""),
                 defaults={
-                    "hook_name": record.get("hook_name", ""),
                     "status": record.get("status", "queued"),
                     "output_str": record.get("output_str", ""),
                 },
@@ -4168,13 +4211,40 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
                         ),
                     )
 
+    def safe_update(self, update_fields: Mapping[str, Any], *, refresh: bool = True) -> bool:
+        """Compare-and-swap one loaded ArchiveResult without opening a transaction."""
+        expected_modified_at = self.modified_at
+        previous_output_size = int(self.output_size or 0)
+        values = dict(update_fields)
+        values.setdefault("modified_at", timezone.now())
+        updated = type(self).objects.filter(pk=self.pk, modified_at=expected_modified_at).update(**values)
+        if updated == 1:
+            for field, value in values.items():
+                setattr(self, field, value)
+            if "output_size" in values:
+                size_delta = int(values["output_size"] or 0) - previous_output_size
+                if size_delta:
+                    Snapshot.objects.filter(pk=self.snapshot_id).update(
+                        output_size=F("output_size") + size_delta,
+                        modified_at=timezone.now(),
+                    )
+        if refresh:
+            try:
+                self.refresh_from_db()
+            except type(self).DoesNotExist:
+                pass
+        return updated == 1
+
     def schedule_delete_cleanup(self, *, using: str | None = None) -> None:
         """Remove shared plugin output and refresh persisted Snapshot metadata after commit."""
         snapshot_id = self.snapshot_id
+        plugin = self.plugin
         paths = self.validate_output_paths_for_delete(self.output_paths_for_delete())
 
         def cleanup() -> None:
-            type(self).delete_output_paths(paths)
+            results = type(self).objects.using(using) if using else type(self).objects
+            if not results.filter(snapshot_id=snapshot_id, plugin=plugin).exists():
+                type(self).delete_output_paths(paths)
             type(self).refresh_snapshot_output_sizes({snapshot_id})
             snapshot = Snapshot.objects.filter(pk=snapshot_id).first()
             if snapshot:
@@ -4208,7 +4278,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
 
     def reset_for_retry(self, *, save: bool = True) -> None:
         self.status = self.StatusChoices.QUEUED
-        self.hook_name = ""
         self.retry_at = None
         self.output_str = ""
         self.output_json = None
@@ -4221,7 +4290,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             self.save(
                 update_fields=[
                     "status",
-                    "hook_name",
                     "retry_at",
                     "output_str",
                     "output_json",
@@ -4276,12 +4344,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             self.refresh_from_db()
         return True
 
-    @property
-    def plugin_module(self) -> Any | None:
-        # Hook scripts are now used instead of Python plugin modules
-        # The plugin name maps to hooks in abx_plugins/plugins/{plugin}/
-        return None
-
     @staticmethod
     def _normalize_output_files(raw_output_files: Any) -> dict[str, dict[str, Any]]:
         from abx_dl.output_files import OutputManifest
@@ -4301,14 +4363,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
     def output_file_paths(self) -> list[str]:
         return list(self.output_file_map().keys())
 
-    def output_file_count(self) -> int:
-        return len(self.output_file_paths())
-
-    def output_size_from_files(self) -> int:
-        from abx_dl.output_files import OutputManifest
-
-        return OutputManifest.from_value(self.output_files).total_size
-
     def update_output_metadata_from_filesystem(self, snapshot_dir: Path | None = None, save: bool = True) -> bool:
         from abx_dl.output_files import OutputManifest, output_file_from_path
 
@@ -4369,9 +4423,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
             self.save(update_fields=["output_files", "output_size", "output_mimetypes", "modified_at"])
         return True
 
-    def output_exists(self) -> bool:
-        return os.path.exists(Path(self.snapshot_dir) / self.plugin)
-
     @staticmethod
     def _looks_like_output_path(raw_output: str | None, plugin_name: str | None = None) -> bool:
         value = str(raw_output or "").strip()
@@ -4470,12 +4521,26 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
                 if Path(candidate).name.lower() == preferred_name:
                     return candidate
 
-        ext_groups = (
-            (".html", ".htm", ".mhtml", ".mht", ".pdf"),
-            (".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
-            (".json", ".jsonl", ".txt", ".md", ".csv", ".tsv"),
-            (".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
-        )
+        plugin_lower = (plugin_name or "").lower()
+        if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl"):
+            # yt-dlp commonly emits a thumbnail plus several media formats.
+            # Prefer something browsers can play directly; otherwise cards can
+            # select a large MKV or thumbnail that the plugin player cannot use.
+            ext_groups = (
+                (".mp4", ".webm", ".m4v", ".ogv"),
+                (".mp3", ".m4a", ".aac", ".opus", ".ogg", ".wav", ".flac"),
+                (".mkv", ".mov", ".avi", ".flv", ".wmv", ".mpg", ".mpeg", ".ts", ".m2ts", ".mts", ".3gp", ".3g2"),
+                (".html", ".htm", ".mhtml", ".mht", ".pdf"),
+                (".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
+                (".json", ".jsonl", ".txt", ".md", ".csv", ".tsv", ".srt", ".vtt"),
+            )
+        else:
+            ext_groups = (
+                (".html", ".htm", ".mhtml", ".mht", ".pdf"),
+                (".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".ico"),
+                (".json", ".jsonl", ".txt", ".md", ".csv", ".tsv"),
+                (".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
+            )
         for ext_group in ext_groups:
             group_candidates = [candidate for candidate in candidates if Path(candidate).suffix.lower() in ext_group]
             if group_candidates:
@@ -4633,50 +4698,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
         process = self.process_record
         return process.timeout if process else 120
 
-    def save_search_index(self):
-        pass
-
-    def _set_binary_from_cmd(self, cmd: list) -> None:
-        """
-        Find Binary for command and set binary FK.
-
-        Tries matching by absolute path first, then by binary name.
-        Only matches binaries on the current machine.
-        """
-        if not cmd:
-            return
-
-        from archivebox.machine.models import Machine
-
-        bin_path_or_name = cmd[0] if isinstance(cmd, list) else cmd
-        machine = Machine.current()
-
-        # Try matching by absolute path first
-        binary = Binary.objects.filter(
-            abspath=bin_path_or_name,
-            machine=machine,
-        ).first()
-
-        if binary:
-            process = self.process_record
-            if process:
-                process.binary = binary
-                process.save()
-            return
-
-        # Fallback: match by binary name
-        bin_name = Path(bin_path_or_name).name
-        binary = Binary.objects.filter(
-            name=bin_name,
-            machine=machine,
-        ).first()
-
-        if binary:
-            process = self.process_record
-            if process:
-                process.binary = binary
-                process.save()
-
     def _url_passes_filters(self, url: str) -> bool:
         """Check if URL passes URL_ALLOWLIST and URL_DENYLIST config filters.
 

archivebox/core/recovery_util.py

@@ -10,6 +10,11 @@ def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
     return exit_code is not None and (exit_code < 0 or exit_code >= 128)
 
 
+def _canonical_hook_name(hook_name: str) -> str:
+    hook_name = Path(hook_name).name
+    return Path(hook_name).stem if Path(hook_name).suffix in {".py", ".js", ".sh"} else hook_name
+
+
 def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
@@ -168,14 +173,23 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
         else:
             orphaned_hook_processes = orphaned_hook_processes.none()
-    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status", "started_at", "ended_at").order_by(
-        "-started_at",
-        "-id",
-    ):
+    for process in orphaned_hook_processes.only(
+        "id",
+        "pwd",
+        "cmd",
+        "process_type",
+        "status",
+        "exit_code",
+        "stdout",
+        "stderr",
+        "started_at",
+        "ended_at",
+    ).order_by("-started_at", "-id"):
         hook_script_name = process.hook_script_name
         if not hook_script_name or not process.pwd:
             continue
         plugin_dir = Path(process.pwd)
+        hook_name = _canonical_hook_name(hook_script_name)
         if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
             continue
         try:
@@ -187,19 +201,20 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             continue
         if snapshot is None:
             continue
-        result, created = ArchiveResult.objects.get_or_create(
-            snapshot=snapshot,
-            plugin=plugin_dir.name,
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_dir.name,
+            hook_name,
             defaults={
-                "hook_name": Path(hook_script_name).stem,
                 "status": ArchiveResult.StatusChoices.QUEUED,
             },
         )
-        if created or result.status == ArchiveResult.StatusChoices.QUEUED:
+        process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts))
+        if result.status == ArchiveResult.StatusChoices.QUEUED or process_is_newer:
             requeue_snapshot = False
             # A runner can die after the hook Process exits but before the
             # ProcessCompletedEvent projector links/finalizes ArchiveResult.
-            # Reconstruct the plugin row from its newest durable Process row.
+            # Reconstruct only that exact hook row from the durable Process row.
             manifest = OutputManifest.scan(plugin_dir, containment_root=snapshot.output_dir)
             output_files = manifest.as_mapping()
             output_size = manifest.total_size
@@ -209,10 +224,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 for record in Process.parse_records_from_text(process.stdout or "")
                 if record.get("type") == "ArchiveResult"
                 and (record.get("plugin") or plugin_dir.name) == plugin_dir.name
-                and (record.get("hook_name") or Path(hook_script_name).stem) == Path(hook_script_name).stem
+                and _canonical_hook_name(str(record.get("hook_name") or hook_name)) == hook_name
             ]
             emitted_result = emitted_records[-1] if emitted_records else {}
-            result.hook_name = Path(hook_script_name).stem
             result.process = process
             result.start_ts = process.started_at
             result.end_ts = process.ended_at
@@ -225,6 +239,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 result.output_size = 0
                 result.output_mimetypes = ""
                 result.output_str = ""
+                result.output_json = None
                 result.status = ArchiveResult.StatusChoices.QUEUED
                 requeue_snapshot = True
             else:
@@ -249,7 +264,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 )
             result.save(
                 update_fields=[
-                    "hook_name",
                     "process",
                     "start_ts",
                     "end_ts",

archivebox/core/tag_util.py

@@ -152,15 +152,8 @@ def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, b
     if not normalized_name:
         raise ValueError("Tag name is required")
 
-    existing = Tag.objects.filter(name__iexact=normalized_name).first()
-    if existing:
-        return existing, False
-
-    tag = Tag.objects.create(
-        name=normalized_name,
-        created_by=created_by,
-    )
-    return tag, True
+    defaults = {"created_by": created_by} if created_by is not None else None
+    return Tag.get_or_create_by_name(normalized_name, defaults=defaults)
 
 
 def rename_tag(tag: Tag, name: str) -> Tag:

archivebox/core/templatetags/core_tags.py

@@ -32,7 +32,7 @@ _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml",
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
 _STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
 
-_MEDIA_FILE_EXTS = {
+_VIDEO_FILE_EXTS = {
     ".mp4",
     ".webm",
     ".mkv",
@@ -49,6 +49,9 @@ _MEDIA_FILE_EXTS = {
     ".3gp",
     ".3g2",
     ".ogv",
+}
+
+_AUDIO_FILE_EXTS = {
     ".mp3",
     ".m4a",
     ".aac",
@@ -66,6 +69,10 @@ _MEDIA_FILE_EXTS = {
     ".dts",
 }
 
+_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
+_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
+_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
+
 
 def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
     from abx_dl.output_files import OutputManifest
@@ -154,15 +161,27 @@ def _list_media_files(result) -> list[dict]:
 
     for rel_path, size in candidates:
         href = str(Path(result.plugin) / rel_path)
+        suffix = rel_path.suffix.lower()
+        media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
         media_files.append(
             {
                 "name": rel_path.name,
                 "path": href,
                 "size": size,
+                "media_type": media_type,
+                "is_video": media_type == "video",
+                "is_audio": media_type == "audio",
+                "is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
             },
         )
 
-    media_files.sort(key=lambda item: item["name"].lower())
+    media_files.sort(
+        key=lambda item: (
+            0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
+            -int(item.get("size") or 0),
+            item["name"].lower(),
+        ),
+    )
     return media_files
 
 

archivebox/core/views.py

@@ -691,26 +691,27 @@ def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[s
 
     plugin = parts[0]
     plugin_relpath = posixpath.join(*parts[1:])
-    result = (
-        ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED)
-        .only("plugin", "output_files")
-        .first()
+    results = list(
+        ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=plugin,
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+        ).only("plugin", "output_files"),
     )
-    if not result:
+    if not results:
         return rel_path, None
-    if not result.output_files:
-        return rel_path, result
 
-    output_files = result.output_files or {}
-    for candidate in (plugin_relpath, rel_path):
-        file_info = output_files.get(candidate)
-        if not isinstance(file_info, dict):
-            continue
-        if file_info.get("root_relative"):
-            return candidate, result
-        return rel_path, result
+    for result in results:
+        output_files = result.output_files or {}
+        for candidate in (plugin_relpath, rel_path):
+            file_info = output_files.get(candidate)
+            if not isinstance(file_info, dict):
+                continue
+            if file_info.get("root_relative"):
+                return candidate, result
+            return rel_path, result
 
-    return rel_path, result
+    return rel_path, results[0]
 
 
 def _plugin_full_preview_response(

archivebox/crawls/models.py

@@ -1191,7 +1191,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 ),
             }
             if tag_names:
-                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                tag_ids = [Tag.get_or_create_by_name(tag_name)[0].pk for tag_name in tag_names]
                 snapshot.add_tag_ids(tag_ids)
 
         existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
@@ -1293,43 +1293,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return created_snapshots
 
-    def install_declared_binaries(self, binary_names: set[str], machine=None) -> None:
-        """Install crawl-declared binaries through their unified lifecycle."""
-        from archivebox.crawls.locks import binary_lifecycle_lock
-        from archivebox.machine.models import Binary, Machine
-
-        if not binary_names:
-            return
-
-        machine = machine or Machine.current()
-        binaries = Binary.objects.filter(machine=machine, name__in=binary_names).order_by("name")
-        for binary in binaries:
-            with binary_lifecycle_lock(str(binary.id)):
-                binary.refresh_from_db()
-                if binary.status == Binary.StatusChoices.INSTALLED:
-                    continue
-                binary.update_and_requeue(retry_at=timezone.now())
-                binary.refresh_from_db()
-                binary.install_claimed(lock_seconds=600)
-
-        unresolved_binaries = list(
-            Binary.objects.filter(
-                machine=machine,
-                name__in=binary_names,
-            )
-            .exclude(
-                status=Binary.StatusChoices.INSTALLED,
-            )
-            .order_by("name"),
-        )
-        if unresolved_binaries:
-            binary_details = ", ".join(
-                f"{binary.name} (status={binary.status}, retry_at={binary.retry_at})" for binary in unresolved_binaries
-            )
-            raise RuntimeError(
-                f"Crawl dependencies failed to install before continuing: {binary_details}",
-            )
-
     def is_finished(self) -> bool:
         """Check if crawl is finished (all snapshots sealed or no snapshots exist)."""
         from archivebox.core.models import Snapshot

archivebox/misc/serve_static.py

@@ -33,6 +33,26 @@ from archivebox.misc.logging_util import printable_filesize
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
 IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
 TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
+html {
+    width: 100%;
+    min-width: 100%;
+    background: #fff;
+}
+body {
+    box-sizing: border-box;
+    width: min(100%, 72rem);
+    max-width: none;
+    min-height: 100vh;
+    margin: 0 auto;
+    padding: clamp(1rem, 3vw, 2rem);
+    background: #fff;
+    color: #111827;
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    line-height: 1.55;
+}
+body > * {
+    max-width: 100%;
+}
 img:not([width]):not([height]) {
     max-width: min(100%, 12rem);
     max-height: 12rem;

archivebox/plugins/discovery.py

@@ -88,70 +88,14 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any
     return get_plugin_config_resolver().enabled_plugin_names_from_flat(dict(config.items()))
 
 
-def discover_plugins_that_provide_interface(
-    module_name: str,
-    required_attrs: list[str],
-    plugin_prefix: str | None = None,
-) -> dict[str, Any]:
-    """
-    Discover plugins that provide a specific Python module with required interface.
-
-    This enables dynamic plugin discovery for features like search backends,
-    storage backends, etc. without hardcoding imports.
-    """
-    import importlib.util
-
-    backends = {}
-
-    for plugin_dir in iter_plugin_dirs():
-        plugin_name = plugin_dir.name
-        if plugin_prefix and not plugin_name.startswith(plugin_prefix):
-            continue
-
-        module_path = plugin_dir / f"{module_name}.py"
-        if not module_path.exists():
-            continue
-
-        try:
-            spec = importlib.util.spec_from_file_location(
-                f"archivebox.dynamic_plugins.{plugin_name}.{module_name}",
-                module_path,
-            )
-            if spec is None or spec.loader is None:
-                continue
-
-            module = importlib.util.module_from_spec(spec)
-            spec.loader.exec_module(module)
-
-            if not all(attr in vars(module) for attr in required_attrs):
-                continue
-
-            if plugin_prefix:
-                backend_name = plugin_name[len(plugin_prefix) :]
-            else:
-                backend_name = plugin_name
-
-            backends[backend_name] = module
-
-        except Exception:
-            continue
-
-    return backends
-
-
-def get_search_backends() -> dict[str, Any]:
-    """
-    Discover all available search backend plugins.
-
-    Search backends must provide a search.py module with:
-        - search(query: str) -> List[str]  (returns snapshot IDs)
-        - flush(snapshot_ids: Iterable[str]) -> None
-    """
-    return discover_plugins_that_provide_interface(
-        module_name="search",
-        required_attrs=["search", "flush"],
-        plugin_prefix="search_backend_",
-    )
+def get_search_backends():
+    """Return plugins that declare both standalone search commands."""
+    catalog = get_plugin_catalog()
+    return {
+        plugin.name.removeprefix("search_backend_"): plugin
+        for plugin in catalog.values()
+        if catalog.command(plugin.name, "search") is not None and catalog.command(plugin.name, "flush") is not None
+    }
 
 
 @lru_cache(maxsize=1)

archivebox/plugins/hooks.py

@@ -8,40 +8,13 @@ from __future__ import annotations
 
 __package__ = "archivebox.plugins"
 
-import os
-from collections.abc import Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Protocol, TypeGuard, runtime_checkable
+from typing import Any
 
-from asgiref.sync import async_to_sync
+from abx_dl.models import parse_hook_filename
 
-from abx_dl.execution import execute_hook
-from abx_dl.models import Hook, parse_hook_filename
-
-from archivebox.config.constants import CONSTANTS
-from archivebox.config.version import VERSION
 from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
-from archivebox.plugins.discovery import ConfigLookup, get_enabled_plugins, get_plugin_catalog, get_plugin_special_config
-
-if TYPE_CHECKING:
-    from archivebox.machine.models import Process
-
-
-@runtime_checkable
-class ConfigDump(Protocol):
-    def as_dict(self) -> dict[str, Any]: ...
-
-
-def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
-    return isinstance(config, ConfigDump)
-
-
-def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
-    if config is None:
-        return {}
-    if _has_config_dump(config):
-        return dict(config.as_dict())
-    return dict(config.items())
+from archivebox.plugins.discovery import ConfigLookup, get_enabled_plugins, get_plugin_catalog
 
 
 def is_background_hook(hook_name: str) -> bool:
@@ -76,149 +49,6 @@ def discover_hooks(
     return [hook.path for _plugin, hook in get_plugin_catalog().hooks(normalized, names=names)]
 
 
-def _catalog_hook(script: Path) -> Hook:
-    script = script.resolve()
-    for plugin in get_plugin_catalog().values():
-        for hook in plugin.hooks:
-            if hook.path.resolve() == script:
-                return hook
-    parsed = parse_hook_filename(script.name)
-    if parsed is None:
-        raise ValueError(f"Not a valid plugin hook filename: {script.name}")
-    event, order, is_background = parsed
-    return Hook(
-        name=script.name,
-        event=event,
-        plugin_name=script.parent.name,
-        path=script,
-        order=order,
-        is_background=is_background,
-    )
-
-
-def _hook_environment(config: ConfigLookup | Mapping[str, Any] | None, **config_scope: Any) -> tuple[dict[str, str], Any]:
-    from archivebox.config.common import (
-        ArchiveBoxConfig,
-        _archivebox_config_input_names,
-        get_config,
-        normalize_runtime_config,
-    )
-
-    overrides = _config_to_overrides(config)
-    resolved = get_config(overrides=overrides, **config_scope)
-    runtime = normalize_runtime_config(
-        resolved.for_crawl_runtime(runtime_overrides=overrides),
-        json_safe=False,
-    )
-    runtime.update(normalize_runtime_config(overrides, json_safe=False))
-
-    env = os.environ.copy()
-    config_input_names = _archivebox_config_input_names()
-    for key in config_input_names:
-        env.pop(key, None)
-    env.pop("PLUGINS", None)
-    env["PATH"] = os.environ.get("PATH", "")
-    env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
-    env["LIBRARY_VERSION"] = VERSION
-    env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
-
-    canonical_config_keys = set(ArchiveBoxConfig.model_fields)
-    for key, value in runtime.items():
-        if key == "PATH" or value is None:
-            continue
-        if key in config_input_names and key not in canonical_config_keys:
-            continue
-        if isinstance(value, bool):
-            env[key] = "true" if value else "false"
-        elif isinstance(value, (dict, list)):
-            import json
-
-            env[key] = json.dumps(value)
-        else:
-            env[key] = str(value)
-
-    node_modules_dir = runtime.get("NODE_MODULES_DIR")
-    lib_dir = runtime.get("ABXPKG_LIB_DIR")
-    if not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
-    if node_modules_dir:
-        env["NODE_MODULES_DIR"] = str(node_modules_dir)
-        env["NODE_MODULE_DIR"] = str(node_modules_dir)
-        node_path = [part for part in str(runtime.get("NODE_PATH") or "").split(os.pathsep) if part]
-        if str(node_modules_dir) not in node_path:
-            node_path.append(str(node_modules_dir))
-        env["NODE_PATH"] = os.pathsep.join(node_path)
-    return env, resolved
-
-
-def run_hook(
-    script: Path,
-    output_dir: Path,
-    config: ConfigLookup | Mapping[str, Any] | None = None,
-    timeout: int | None = None,
-    parent: Process | None = None,
-    **kwargs: Any,
-) -> Process:
-    """Compatibility adapter for finite direct calls; abx-dl owns execution."""
-    from archivebox.machine.models import Process
-    from archivebox.services.process_service import ProcessService as PersistedProcessService
-    from archivebox.services.process_service import parse_event_datetime
-    from abx_dl.orchestrator import create_bus
-
-    # Preserve the old direct-call contract: hooks are children of an explicit
-    # parent, or of the current ArchiveBox process when one can be identified.
-    # This belongs on the DB projection adapter, not in hook CLI arguments.
-    if parent is None:
-        try:
-            parent = Process.current()
-        except Exception:
-            parent = None
-    config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
-    env, resolved = _hook_environment(config, **config_scope)
-    hook = _catalog_hook(script)
-    if timeout is None:
-        timeout = get_plugin_special_config(hook.plugin_name, resolved)["timeout"]
-    timeout = min(int(timeout or 300), int(CONSTANTS.MAX_HOOK_RUNTIME_SECONDS))
-
-    bus = create_bus(name=f"ArchiveBoxHook_{hook.plugin_name}", total_timeout=float(timeout) + 30.0)
-    PersistedProcessService(bus, parent_process_id=str(parent.id) if parent is not None else None)
-
-    async def execute_and_close():
-        try:
-            return await execute_hook(
-                hook,
-                output_dir=output_dir,
-                env=env,
-                arguments=kwargs,
-                timeout=timeout,
-                bus=bus,
-                process_type=Process.TypeChoices.HOOK,
-            )
-        finally:
-            await bus.wait_until_idle()
-            await bus.destroy(clear=False)
-
-    completed = async_to_sync(execute_and_close)()
-    started_at = parse_event_datetime(completed.start_ts)
-    process = Process.objects.filter(pid=completed.pid or None, started_at=started_at).order_by("-modified_at").first()
-    if process is None:
-        raise RuntimeError(f"Hook {hook.full_name} completed without an ArchiveBox Process projection")
-    return process
-
-
-def extract_records_from_process(process: Process) -> list[dict[str, Any]]:
-    """Return hook JSONL records with generic catalog identity attached."""
-    records = process.get_records()
-    plugin_name = Path(process.pwd).name if process.pwd else "unknown"
-    plugin_hook = next((str(arg) for arg in process.cmd if Path(str(arg)).name.startswith("on_")), "")
-    hook_name = Path(plugin_hook).name if plugin_hook else "unknown"
-    for record in records:
-        record.setdefault("plugin", plugin_name)
-        record.setdefault("hook_name", hook_name)
-        record.setdefault("plugin_hook", plugin_hook)
-    return records
-
-
 def collect_urls_from_plugins(snapshot_dir: Path) -> list[dict[str, Any]]:
     """Read the durable urls.jsonl interface emitted by parser plugins."""
     urls: list[dict[str, Any]] = []

archivebox/progressmonitor/views.py

@@ -30,12 +30,16 @@ def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_
 @lru_cache(maxsize=1)
 def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
     plugin_configs = discover_plugin_configs()
+    indexing_plugin_names = frozenset(
+        plugin_name
+        for plugin_name, plugin_config in plugin_configs.items()
+        if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
+    )
     download_plugin_names = frozenset(
         plugin_name
         for plugin_name, plugin_config in plugin_configs.items()
-        if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
+        if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
     )
-    indexing_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_"))
     return download_plugin_names, indexing_plugin_names
 
 

archivebox/search/backends.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.search"
 
+import json
 import os
-from contextlib import contextmanager
 from typing import Any
 
 from archivebox.config.common import get_config
@@ -10,29 +10,21 @@ from archivebox.config.common import get_config
 _search_backends_cache: dict | None = None
 
 
-@contextmanager
-def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
-    """Temporarily expose resolved search config through os.environ for backend code."""
+def search_backend_command_env(config: dict[str, Any] | None = None, **config_kwargs: Any) -> dict[str, str]:
+    """Serialize resolved application config for a standalone plugin command."""
     config = config or get_config(**config_kwargs)
-    updates = {}
+    env = os.environ.copy()
     for key, value in config.items():
         key = str(key)
-        if not (key.startswith("SEARCH_BACKEND_") or key.endswith("_BINARY")):
-            continue
         if value is None:
             continue
-        if isinstance(value, (str, int, float, bool, os.PathLike)):
-            updates[key] = str(value)
-    previous = {key: os.environ.get(key) for key in updates}
-    os.environ.update(updates)
-    try:
-        yield
-    finally:
-        for key, value in previous.items():
-            if value is None:
-                os.environ.pop(key, None)
-            else:
-                os.environ[key] = value
+        if isinstance(value, bool):
+            env[key] = "true" if value else "false"
+        elif isinstance(value, (dict, list, tuple)):
+            env[key] = json.dumps(value)
+        elif isinstance(value, (str, int, float, os.PathLike)):
+            env[key] = str(value)
+    return env
 
 
 def normalize_search_backend_name(backend_name: str | None) -> str:
@@ -41,7 +33,7 @@ def normalize_search_backend_name(backend_name: str | None) -> str:
 
 
 def get_available_backends() -> dict:
-    """Discover search backend plugin modules and cache them in memory."""
+    """Discover search-capable plugins and cache their catalog entries."""
     global _search_backends_cache
 
     if _search_backends_cache is None:
@@ -53,7 +45,7 @@ def get_available_backends() -> dict:
 
 
 def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any:
-    """Resolve the configured search backend module."""
+    """Resolve the configured search-capable plugin."""
     config = config or get_config(**config_kwargs)
     backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
     backends = get_available_backends()

archivebox/search/query.py

@@ -8,7 +8,7 @@ from django.db.models import Case, IntegerField, Q, QuerySet, Value, When
 from archivebox.config.common import get_config
 from archivebox.misc.logging import stderr
 from archivebox.misc.util import enforce_types
-from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_env
+from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_command_env
 from archivebox.search.config import get_search_mode, get_search_mode_backend, get_search_mode_base
 
 
@@ -278,18 +278,27 @@ def iter_query_search_ids(
                     # --csv/--json. Keep daemon diagnostics on stderr.
                     with redirect_stdout(sys.stderr):
                         ensure_daemon_stack(reason="search query")
-                with search_backend_env(config=config):
-                    if backend_name == "ripgrep":
-                        ids = backend.iter_search(query, search_mode=search_mode_base)
-                    else:
-                        ids = backend.search(query)
-                    for snapshot_id in ids:
-                        if snapshot_id in seen:
-                            continue
-                        seen.add(snapshot_id)
-                        yield snapshot_id
-                        if max_results and len(seen) >= max_results:
-                            return
+                from abx_dl.execution import iter_plugin_command
+                from archivebox.config.constants import CONSTANTS
+                from archivebox.plugins.discovery import get_plugin_catalog
+
+                command = get_plugin_catalog().command(backend.name, "search")
+                if command is None:
+                    raise RuntimeError(f'Plugin "{backend.name}" does not expose a search command')
+                ids = iter_plugin_command(
+                    command,
+                    arguments={"query": query, "search_mode": search_mode_base},
+                    env=search_backend_command_env(config=config),
+                    cwd=CONSTANTS.DATA_DIR,
+                    timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+                )
+                for snapshot_id in ids:
+                    if snapshot_id in seen:
+                        continue
+                    seen.add(snapshot_id)
+                    yield snapshot_id
+                    if max_results and len(seen) >= max_results:
+                        return
                 successful_backends += 1
             except Exception as err:
                 errors.append(err)
@@ -318,8 +327,22 @@ def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None
     snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)]
 
     try:
-        with search_backend_env(config=config):
-            backend.flush(snapshot_pks)
+        from abx_dl.execution import iter_plugin_command
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.plugins.discovery import get_plugin_catalog
+
+        command = get_plugin_catalog().command(backend.name, "flush")
+        if command is None:
+            raise RuntimeError(f'Plugin "{backend.name}" does not expose a flush command')
+        list(
+            iter_plugin_command(
+                command,
+                stdin=snapshot_pks,
+                env=search_backend_command_env(config=config),
+                cwd=CONSTANTS.DATA_DIR,
+                timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+            ),
+        )
     except Exception as err:
         stderr()
         stderr(

archivebox/services/archive_result_service.py

@@ -4,7 +4,6 @@ import asyncio
 import inspect
 import json
 import os
-import re
 import signal
 import sys
 import time
@@ -14,7 +13,6 @@ from pathlib import Path
 from typing import Any
 
 from asgiref.sync import sync_to_async
-from django.db import IntegrityError
 from django.utils import timezone
 
 from abx_dl.events import PROCESS_EXIT_SKIPPED, ArchiveResultEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent
@@ -87,11 +85,6 @@ def _normalize_status(status: str) -> str:
     return status or "failed"
 
 
-def _snapshot_hook_order(hook_name: str) -> int:
-    match = re.match(r"^on_Snapshot__(\d+)", hook_name or "")
-    return int(match.group(1)) if match else -1
-
-
 def _normalize_snapshot_title(candidate: str, *, snapshot_url: str) -> str:
     title = " ".join(line.strip() for line in str(candidate or "").splitlines() if line.strip()).strip()
     if not title:
@@ -209,7 +202,6 @@ def _save_archiveresult_event_to_db(
         start_ts = parse_event_datetime(event.start_ts)
         end_ts = parse_event_datetime(event.end_ts) or timezone.now()
         defaults = {
-            "hook_name": event.hook_name,
             "status": _normalize_status(event.status),
             "output_str": event.output_str,
             "output_json": event.output_json,
@@ -224,48 +216,13 @@ def _save_archiveresult_event_to_db(
         if event.error:
             defaults["notes"] = event.error
 
-    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
-        result = ArchiveResult.objects.filter(
-            snapshot=snapshot,
-            plugin=event.plugin,
-        ).first()
-    if result is None:
-        try:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
-                result = ArchiveResult.objects.create(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                    **defaults,
-                )
-        except IntegrityError:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
-                result = ArchiveResult.objects.get(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                )
-
-    if result.output_files:
-        merged_output_files = {**result.output_files, **defaults["output_files"]}
-        defaults["output_files"], defaults["output_size"], defaults["output_mimetypes"] = _manifest_metadata(
-            OutputManifest.from_value(merged_output_files),
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_or_create"):
+        result, _created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            event.plugin,
+            event.hook_name,
+            defaults=defaults,
         )
-        defaults["output_size"] = max(defaults["output_size"], int(result.output_size or 0))
-        defaults["output_mimetypes"] = ",".join(
-            dict.fromkeys(
-                mimetype.strip()
-                for value in (defaults["output_mimetypes"], result.output_mimetypes)
-                for mimetype in value.split(",")
-                if mimetype.strip()
-            ),
-        )
-    if result.start_ts and defaults["start_ts"]:
-        defaults["start_ts"] = min(result.start_ts, defaults["start_ts"])
-    if result.end_ts and defaults["end_ts"]:
-        defaults["end_ts"] = max(result.end_ts, defaults["end_ts"])
-    if _snapshot_hook_order(result.hook_name) > _snapshot_hook_order(event.hook_name):
-        for field in ("hook_name", "status", "output_str", "output_json", "process_id", "notes"):
-            if field in result.__dict__:
-                defaults[field] = result.__dict__[field]
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
@@ -307,7 +264,7 @@ def _save_archiveresult_event_to_db(
 
 
 def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
-    """Advance an existing queued plugin row after its OS process is persisted."""
+    """Advance an existing queued hook row after its OS process is persisted."""
     from archivebox.core.models import ArchiveResult
 
     started_at = parse_event_datetime(event.start_ts)
@@ -316,9 +273,9 @@ def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str,
     ArchiveResult.objects.filter(
         snapshot_id=snapshot_id,
         plugin=event.plugin_name,
+        hook_name=event.hook_name,
         status=ArchiveResult.StatusChoices.QUEUED,
     ).update(
-        hook_name=event.hook_name,
         status=ArchiveResult.StatusChoices.STARTED,
         start_ts=started_at,
         end_ts=None,
@@ -333,7 +290,7 @@ class ArchiveResultService(BaseService):
 
     def __init__(self, bus):
         self._completed_process_event_ids: set[str] = set()
-        self._save_locks: dict[tuple[str, str], asyncio.Lock] = {}
+        self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
         super().__init__(bus)
         self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
         self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
@@ -348,7 +305,7 @@ class ArchiveResultService(BaseService):
                 where=lambda candidate: self.bus.event_is_child_of(event, candidate),
             )
 
-        key = (str(event.snapshot_id), event.plugin)
+        key = (str(event.snapshot_id), event.plugin, event.hook_name)
         lock = self._save_locks.setdefault(key, asyncio.Lock())
         async with lock:
             await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started)

archivebox/services/process_service.py

@@ -43,10 +43,10 @@ def normalize_process_env(env: dict) -> dict:
         if is_sensitive_config_key(key) or (key in config_input_names and key not in allowed_config_keys):
             normalized.pop(key, None)
     if selected_plugins:
-        from abx_dl.models import discover_plugins, filter_plugins
         from archivebox.config.common import _plugin_enabled_config_keys
+        from archivebox.plugins.discovery import get_plugin_catalog
 
-        selected_plugins = set(filter_plugins(discover_plugins(runtime="archivebox"), sorted(selected_plugins), include_providers=True))
+        selected_plugins = set(get_plugin_catalog().select(sorted(selected_plugins)))
         for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
             normalized.setdefault(enabled_key, "True" if plugin_name in selected_plugins else "False")
     return normalized
@@ -61,12 +61,8 @@ class ProcessService(BaseService):
     ]
     EMITS: ClassVar[list[type[BaseEvent]]] = []
 
-    def __init__(self, bus, *, parent_process_id: str | None = None):
+    def __init__(self, bus):
         self._iface = None
-        # A direct run_hook() call owns a private bus, so its caller-supplied
-        # parent applies to every process projected from that bus. Crawl buses
-        # leave this unset and derive their hierarchy from lifecycle events.
-        self.parent_process_id = parent_process_id
         self._completed_queue: asyncio.Queue[ProcessCompletedEvent | None] = asyncio.Queue()
         self._completed_worker: asyncio.Task | None = None
         super().__init__(bus)
@@ -104,7 +100,7 @@ class ProcessService(BaseService):
             process = await Process.objects.acreate(
                 machine=iface.machine,
                 iface=iface,
-                parent_id=self.parent_process_id,
+                parent_id=None,
                 process_type=process_type,
                 worker_type=worker_type,
                 pwd=event.output_dir,
@@ -138,7 +134,7 @@ class ProcessService(BaseService):
             hook_path=event.hook_path,
         )
         await Process.objects.filter(id=process.id).aupdate(
-            parent_id=self.parent_process_id or process.parent_id,
+            parent_id=process.parent_id,
             pwd=process.pwd,
             cmd=process.cmd,
             env=process.env,
@@ -225,7 +221,7 @@ class ProcessService(BaseService):
             await Process.objects.acreate(
                 machine=iface.machine,
                 iface=iface,
-                parent_id=self.parent_process_id,
+                parent_id=None,
                 process_type=process_type,
                 worker_type=worker_type,
                 pwd=event.output_dir,
@@ -246,7 +242,7 @@ class ProcessService(BaseService):
         updates = {
             "machine_id": iface.machine_id,
             "iface_id": iface.id,
-            "parent_id": self.parent_process_id or process.parent_id,
+            "parent_id": process.parent_id,
             "pwd": event.output_dir,
             "env": process_env,
             "pid": event.pid or process.pid,

archivebox/services/runner.py

@@ -28,22 +28,19 @@ from abx_dl.events import (
     CrawlEvent,
     CrawlSetupEvent,
     CrawlStartEvent,
-    MachineEvent,
     ProcessCompletedEvent,
     ProcessEvent,
     SnapshotCompletedEvent,
     SnapshotEvent,
     slow_warning_timeout,
 )
-from abx_dl.heartbeat import CrawlHeartbeat
 from abx_dl.limits import CrawlLimitState
 from abx_dl.catalog import PluginCatalog
-from abx_dl.models import Plugin, Snapshot as AbxSnapshot, filter_plugins
+from abx_dl.models import Snapshot as AbxSnapshot
 from abx_dl.orchestrator import (
     ExecutionPlan,
     create_bus,
     install_plugins as abx_install_plugins,
-    setup_services as setup_abx_services,
 )
 from abx_dl.services.process_service import ProcessService as HookProcessService
 from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService
@@ -119,8 +116,8 @@ def _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: st
     Console(highlight=False).print(line)
 
 
-def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str] | None) -> int:
-    selected = filter_plugins(plugins, selected_plugins) if selected_plugins else plugins
+def _count_selected_hooks(catalog: PluginCatalog, selected_plugins: list[str] | None) -> int:
+    selected = catalog.select(selected_plugins) if selected_plugins else catalog
     return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
 
 
@@ -128,14 +125,6 @@ def _is_nonfatal_setup_hook(plugin_name: str, hook_name: str) -> bool:
     return plugin_name == "chrome" and hook_name.endswith("_chrome_kill_zombies")
 
 
-def _discover_archivebox_plugins() -> dict[str, Plugin]:
-    return _discover_archivebox_catalog().plugins
-
-
-def _discover_archivebox_catalog() -> PluginCatalog:
-    return get_plugin_catalog()
-
-
 def _runner_task_context() -> contextvars.Context:
     context = contextvars.copy_context()
     context.run(EventBus.current_event_context.set, None)
@@ -148,33 +137,6 @@ def _is_external_task_cancelled(error: asyncio.CancelledError) -> bool:
     return not isinstance(error, (EventHandlerAbortedError, EventHandlerCancelledError))
 
 
-async def _emit_machine_config(
-    bus,
-    *,
-    config: dict[str, Any],
-    derived_config: dict[str, Any],
-    parent_event=None,
-) -> None:
-    user_config = normalize_runtime_config(config)
-    user_config["ABX_RUNTIME"] = "archivebox"
-    derived_machine_config = normalize_runtime_config(derived_config)
-    user_event = MachineEvent(
-        config=user_config,
-        config_type="user",
-    )
-    if parent_event is not None:
-        user_event.event_parent_id = parent_event.event_id
-    await bus.emit(user_event).now()
-    if derived_machine_config:
-        derived_event = MachineEvent(
-            config=derived_machine_config,
-            config_type="derived",
-        )
-        if parent_event is not None:
-            derived_event.event_parent_id = parent_event.event_id
-        await bus.emit(derived_event).now()
-
-
 async def _run_event_now(event, timeout: float | None = None):
     await event.now(timeout=timeout)
     await event.wait(timeout=timeout)
@@ -222,8 +184,7 @@ class CrawlRunner:
     ):
         self.crawl = crawl
         self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
-        self.catalog = _discover_archivebox_catalog()
-        self.plugins = self.catalog.plugins
+        self.catalog = get_plugin_catalog()
         HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts)
         register_sonic_daemon_event_handler(self.bus)
         PersistedProcessService(self.bus)
@@ -290,10 +251,9 @@ class CrawlRunner:
         if self.allow_maintenance_on_inactive_crawl:
             # SEALED is the normal terminal state of a finished crawl, not a
             # cancellation signal for maintenance work on its already-sealed
-            # snapshots (search backend backfill, fs migration, etc.). When the
-            # runner is invoked with explicit snapshot_ids + selected_plugins,
-            # treat sealed as completed rather than cancelled so the requested
-            # maintenance hooks can actually run.
+            # snapshots (plugin backfill, fs migration, etc.). When the runner
+            # is given explicit snapshot_ids + selected_plugins, the caller has
+            # deliberately scoped work that remains valid after crawl sealing.
             return False
         return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists()
 
@@ -312,25 +272,14 @@ class CrawlRunner:
             await _run_event_now(abort_event, abort_event.event_timeout)
             return
 
-    def runtime_plugins(self) -> dict[str, Plugin]:
-        return self.catalog.select(self.selected_plugins).plugins if self.selected_plugins else self.plugins
-
     @property
     def allow_maintenance_on_inactive_crawl(self) -> bool:
-        """Run targeted search indexing on an already-sealed snapshot.
-
-        This is the singular hook-execution exception to the unified lifecycle.
-        All other work must queue the Snapshot and Crawl normally.
-        """
+        """Run explicitly targeted plugin work on already-sealed snapshots."""
         return bool(
-            self.initial_snapshot_ids
-            and self.selected_plugins
-            and self.crawl.status == self.crawl.StatusChoices.SEALED
-            and all(plugin.startswith("search_backend_") for plugin in self.selected_plugins),
+            self.initial_snapshot_ids and self.selected_plugins and self.crawl.status == self.crawl.StatusChoices.SEALED,
         )
 
     async def run(self) -> None:
-        heartbeat: CrawlHeartbeat | None = None
         root_snapshot_id: str | None = None
         bus_destroyed = False
         try:
@@ -353,25 +302,17 @@ class CrawlRunner:
                 raise_on_first_signal=False,
             ):
                 snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
-                heartbeat = CrawlHeartbeat(
-                    Path(self.crawl_output_dir),
-                    runtime="archivebox",
-                    crawl_id=str(self.crawl.id),
-                )
                 max_concurrent_snapshots = max(1, int(self.base_config.get("CRAWL_MAX_CONCURRENT_SNAPSHOTS", 1)))
                 self.max_concurrent_snapshots = max_concurrent_snapshots
                 self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
                 live_ui = self._create_live_ui()
                 with live_ui if live_ui is not None else nullcontext():
                     try:
-                        await heartbeat.start()
                         if snapshot_ids:
                             root_snapshot_id = snapshot_ids[0]
                             await self.run_crawl(root_snapshot_id, snapshot_ids)
                     finally:
                         self._run_task = None
-                        if heartbeat is not None:
-                            await heartbeat.stop()
                         await self.stop_snapshot_tasks()
                         try:
                             await self.bus.wait_until_idle(timeout=1.0 if self._skip_wait_until_idle else 30.0)
@@ -383,8 +324,6 @@ class CrawlRunner:
         finally:
             if not bus_destroyed:
                 self._run_task = None
-                if heartbeat is not None:
-                    await heartbeat.stop()
                 await self.stop_snapshot_tasks()
                 await self.bus.destroy(clear=False)
             if self._live_stream is not None:
@@ -615,7 +554,7 @@ class CrawlRunner:
             return []
         if self.initial_snapshot_ids:
             # Explicit ids select normal runnable work, except for the one
-            # sealed-search backfill admitted by allow_maintenance_on_inactive_crawl.
+            # targeted maintenance admitted by allow_maintenance_on_inactive_crawl.
             return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids]
         pending_snapshots = list(
             self.crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES)
@@ -764,10 +703,10 @@ class CrawlRunner:
                 "LINES": str(terminal_height),
             },
         )
-        plugins_label = ", ".join(self.selected_plugins) if self.selected_plugins else f"all ({len(self.plugins)} available)"
+        plugins_label = ", ".join(self.selected_plugins) if self.selected_plugins else f"all ({len(self.catalog)} available)"
         live_ui = LiveBusUI(
             self.bus,
-            total_hooks=_count_selected_hooks(self.plugins, self.selected_plugins),
+            total_hooks=_count_selected_hooks(self.catalog, self.selected_plugins),
             timeout_seconds=self.base_config["TIMEOUT"],
             ui_console=ui_console,
             interactive_tty=interactive_tty,
@@ -824,7 +763,7 @@ class CrawlRunner:
         normalized_config = normalize_runtime_config(config)
         configured_plugins = [name.strip().lower() for name in str(normalized_config.get("PLUGINS") or "").split(",") if name.strip()]
         if configured_plugins:
-            selected_plugin_names = set(filter_plugins(self.plugins, configured_plugins, include_providers=True))
+            selected_plugin_names = set(self.catalog.select(configured_plugins))
             for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
                 normalized_config.setdefault(enabled_key, plugin_name in selected_plugin_names)
         return {
@@ -859,7 +798,7 @@ class CrawlRunner:
             derived_config=derived_config,
             runtime="archivebox",
         )
-        setup_hooks = [(plugin, hook) for plugin in plan.plugins.values() for hook in plugin.filter_hooks("CrawlSetup")]
+        setup_hooks = [(plugin, hook) for plugin in plan.catalog.values() for hook in plugin.filter_hooks("CrawlSetup")]
         abx_snapshot = AbxSnapshot(
             id=snapshot["id"],
             url=snapshot["url"],
@@ -878,7 +817,17 @@ class CrawlRunner:
             + CrawlCompletedEvent.model_fields["event_timeout"].default
             + 30.0
         )
-        await plan.seed_config(self.bus)
+        if plan.catalog:
+            await abx_install_plugins(
+                plan,
+                output_dir=output_dir,
+                emit_jsonl=False,
+                bus=self.bus,
+                BinaryService=None,
+                ProcessService=None,
+            )
+        else:
+            await plan.seed_config(self.bus)
         plan.attach_services(
             self.bus,
             url=snapshot["url"],
@@ -895,6 +844,7 @@ class CrawlRunner:
             emit_jsonl=False,
             abort_requested=self.crawl_is_cancelled,
             PluginBinariesService=None,
+            PluginBinaryEnvService=None,
             BinaryService=None,
             ProcessService=None,
             ArchiveResultService=None,
@@ -1053,12 +1003,10 @@ class CrawlRunner:
             def queued_plugins_selected_by_config(queued_plugins: list[str]) -> list[str]:
                 if not snapshot_selected_plugins:
                     return queued_plugins
-                expanded_selected_plugins = set(
-                    filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True).keys(),
-                )
+                expanded_selected_plugins = set(self.catalog.select(snapshot_selected_plugins))
                 return [plugin for plugin in queued_plugins if plugin in expanded_selected_plugins]
 
-            queued_plugins = None
+            selected_hooks_by_plugin = None
             if snapshot["status"] == "started":
                 _reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)()
                 if running_count:
@@ -1076,8 +1024,8 @@ class CrawlRunner:
                     )
                     return
                 if not self.selected_plugins_from_args:
-                    queued_plugins = await sync_to_async(
-                        queued_plugins_for_snapshot,
+                    queued_plugins, selected_hooks_by_plugin = await sync_to_async(
+                        queued_plugins_and_hooks_for_snapshot,
                         thread_sensitive=True,
                     )(snapshot["id"])
                     if queued_plugins:
@@ -1090,10 +1038,13 @@ class CrawlRunner:
                                     snapshot["id"],
                                     disabled_queued_plugins,
                                 )
+                            selected_hooks_by_plugin = {
+                                plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
+                            }
                         snapshot_selected_plugins = queued_plugins
             elif not self.selected_plugins_from_args:
-                queued_plugins = await sync_to_async(
-                    queued_plugins_for_snapshot,
+                queued_plugins, selected_hooks_by_plugin = await sync_to_async(
+                    queued_plugins_and_hooks_for_snapshot,
                     thread_sensitive=True,
                 )(snapshot["id"])
                 if queued_plugins:
@@ -1106,6 +1057,9 @@ class CrawlRunner:
                                 snapshot["id"],
                                 disabled_queued_plugins,
                             )
+                        selected_hooks_by_plugin = {
+                            plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
+                        }
                     snapshot_selected_plugins = queued_plugins
             if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in (
                 "crawl_max_size",
@@ -1115,15 +1069,11 @@ class CrawlRunner:
                 return
             derived_config = normalize_runtime_config(self.derived_config)
             output_dir = Path(snapshot["output_dir"])
-            plugins = (
-                filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
-                if snapshot_selected_plugins
-                else self.plugins
-            )
-            if queued_plugins is not None:
-                await sync_to_async(fail_unavailable_queued_plugins, thread_sensitive=True)(
+            plugins = self.catalog.select(snapshot_selected_plugins) if snapshot_selected_plugins else self.catalog
+            if selected_hooks_by_plugin is not None:
+                await sync_to_async(fail_unavailable_queued_hooks, thread_sensitive=True)(
                     snapshot["id"],
-                    queued_plugins,
+                    selected_hooks_by_plugin,
                     plugins,
                 )
                 remaining_queued_plugins = await sync_to_async(
@@ -1142,7 +1092,8 @@ class CrawlRunner:
                         await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
                 snapshot_selected_plugins = remaining_queued_plugins
-                plugins = filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
+                plugins = self.catalog.select(snapshot_selected_plugins)
+                selected_hooks_by_plugin = include_background_prerequisite_hooks(selected_hooks_by_plugin, plugins)
             abx_snapshot = AbxSnapshot(
                 id=snapshot["id"],
                 url=snapshot["url"],
@@ -1156,7 +1107,7 @@ class CrawlRunner:
                 derived_config=derived_config,
                 runtime="archivebox",
             )
-            plugins = plan.plugins
+            plugins = plan.catalog
             snapshot_phase_timeout = plan.snapshot_timeout + 120.0
             await plan.seed_config(self.bus, parent_event=crawl_start_event)
             snapshot_service = plan.attach_snapshot_service(
@@ -1167,7 +1118,7 @@ class CrawlRunner:
                 snapshot_service=HookSnapshotService,
                 timeout_padding=120.0,
                 abort_requested=self.crawl_is_cancelled,
-                selected_hooks_by_plugin=None,
+                selected_hooks_by_plugin=selected_hooks_by_plugin,
                 emit_discovered_snapshot_events=False,
             )
             try:
@@ -1323,7 +1274,6 @@ async def _run_binary(binary_id: str) -> None:
     from archivebox.machine.models import Binary, Machine
 
     binary = await Binary.objects.aget(id=binary_id)
-    plugins = _discover_archivebox_plugins()
     config = get_config(include_machine=False)
     machine = await sync_to_async(Machine.current, thread_sensitive=True)()
     derived_config = normalize_runtime_config(machine.config)
@@ -1336,9 +1286,16 @@ async def _run_binary(binary_id: str) -> None:
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)
-    setup_abx_services(
+    catalog = get_plugin_catalog()
+    plan = ExecutionPlan.build(
+        catalog,
+        selected_plugins=list(catalog),
+        config=config,
+        derived_config=derived_config,
+        runtime="archivebox",
+    )
+    plan.attach_services(
         bus,
-        plugins=plugins,
         install_enabled=False,
         crawl_setup_enabled=False,
         crawl_start_enabled=False,
@@ -1348,7 +1305,7 @@ async def _run_binary(binary_id: str) -> None:
         emit_jsonl=False,
         BinaryService=None,
     )
-    await _emit_machine_config(bus, config=config, derived_config=derived_config)
+    await plan.seed_config(bus)
 
     try:
         await bus.emit(
@@ -1375,56 +1332,82 @@ def run_binary(binary_id: str) -> None:
     asyncio.run(_run_binary(binary_id))
 
 
-def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
+def queued_plugins_and_hooks_for_snapshot(snapshot_id: str) -> tuple[list[str] | None, dict[str, set[str] | None] | None]:
     from archivebox.core.models import ArchiveResult
 
-    queued_plugins = list(
+    queued_results = list(
         ArchiveResult.objects.filter(
             snapshot_id=snapshot_id,
             status=ArchiveResult.StatusChoices.QUEUED,
         )
         .exclude(plugin="")
-        .order_by("plugin")
-        .values_list("plugin", flat=True),
+        .only("id", "plugin", "hook_name"),
     )
-    return queued_plugins or None
+
+    selected_hooks_by_plugin: dict[str, set[str] | None] = {}
+    queued_plugins = sorted({result.plugin for result in queued_results})
+    for result in queued_results:
+        # hook_name is the modern scheduler identity. Empty hook_name rows are
+        # legacy plugin-level work and must keep running the whole plugin.
+        if not result.hook_name:
+            selected_hooks_by_plugin[result.plugin] = None
+        elif result.plugin not in selected_hooks_by_plugin:
+            selected_hooks_by_plugin[result.plugin] = {result.hook_name}
+        elif selected_hooks_by_plugin[result.plugin] is not None:
+            selected_hooks_by_plugin[result.plugin].add(result.hook_name)
+    if queued_plugins:
+        return queued_plugins, selected_hooks_by_plugin
+    return None, None
+
+
+def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
+    queued_plugins, _selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(snapshot_id)
+    return queued_plugins
 
 
 def config_overrides_for_queued_plugins(selected_plugins: list[str], **overrides: Any) -> dict[str, Any]:
     config_overrides = dict(overrides)
     config_overrides["PLUGINS"] = ",".join(selected_plugins)
-    selected_plugin_names = set(
-        filter_plugins(_discover_archivebox_plugins(), [plugin_name.lower() for plugin_name in selected_plugins], include_providers=True),
-    )
+    selected_plugin_names = set(get_plugin_catalog().select(plugin_name.lower() for plugin_name in selected_plugins))
     for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
         config_overrides[enabled_key] = plugin_name in selected_plugin_names
-    for plugin_name in selected_plugins:
-        if plugin_name.startswith("search_backend_"):
-            config_overrides[f"{plugin_name.upper()}_ENABLED"] = True
     return config_overrides
 
 
-def fail_unavailable_queued_plugins(
+def fail_unavailable_queued_hooks(
     snapshot_id: str,
-    queued_plugins: list[str],
-    plugins: dict[str, Plugin],
+    selected_hooks_by_plugin: dict[str, set[str] | None],
+    plugins: PluginCatalog,
 ) -> None:
     from archivebox.core.models import ArchiveResult
 
-    unavailable_plugins = set(queued_plugins) - set(plugins)
-    if not unavailable_plugins:
-        return
     now = timezone.now()
-    ArchiveResult.objects.filter(
-        snapshot_id=snapshot_id,
-        plugin__in=unavailable_plugins,
-        status=ArchiveResult.StatusChoices.QUEUED,
-    ).update(
-        status=ArchiveResult.StatusChoices.FAILED,
-        start_ts=now,
-        end_ts=now,
-        output_str="Queued plugin is not available in this ArchiveBox installation",
-    )
+    for plugin_name, selected_hook_names in selected_hooks_by_plugin.items():
+        if selected_hook_names is None:
+            continue
+        if plugin_name in plugins:
+            available_hook_names = {
+                name for hook in plugins[plugin_name].filter_hooks("Snapshot") for name in (hook.name, Path(hook.name).stem)
+            }
+        else:
+            available_hook_names = set()
+        missing_hook_names = [hook_name for hook_name in selected_hook_names if hook_name not in available_hook_names]
+        if not missing_hook_names:
+            continue
+        # Hook-level resume rows are durable scheduler state. If a plugin is
+        # installed but no longer exposes a queued hook, mark that row failed so
+        # the snapshot is not retried forever with no hook left to execute.
+        ArchiveResult.objects.filter(
+            snapshot_id=snapshot_id,
+            plugin=plugin_name,
+            hook_name__in=missing_hook_names,
+            status=ArchiveResult.StatusChoices.QUEUED,
+        ).update(
+            status=ArchiveResult.StatusChoices.FAILED,
+            start_ts=now,
+            end_ts=now,
+            output_str="Queued hook is no longer available in the installed plugin",
+        )
 
 
 def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> None:
@@ -1449,6 +1432,35 @@ def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> N
     )
 
 
+def include_background_prerequisite_hooks(
+    selected_hooks_by_plugin: dict[str, set[str] | None],
+    plugins: PluginCatalog,
+) -> dict[str, set[str] | None]:
+    expanded: dict[str, set[str] | None] = {}
+    for plugin_name, selected_hook_names in selected_hooks_by_plugin.items():
+        if selected_hook_names is None or plugin_name not in plugins:
+            expanded[plugin_name] = selected_hook_names
+            continue
+        plugin_hooks = sorted(plugins[plugin_name].filter_hooks("Snapshot"), key=lambda hook: hook.sort_key)
+        selected_sort_keys = [
+            hook.sort_key for hook in plugin_hooks if hook.name in selected_hook_names or Path(hook.name).stem in selected_hook_names
+        ]
+        if not selected_sort_keys:
+            expanded[plugin_name] = set(selected_hook_names)
+            continue
+        first_selected_sort_key = min(selected_sort_keys)
+        expanded_hook_names = set(selected_hook_names)
+        # Earlier background hooks publish live resources (e.g. Chrome tabs)
+        # needed by later foreground hooks, but completed foreground hooks stay
+        # final and are not rerun during hook-level resume.
+        for hook in plugin_hooks:
+            if hook.is_background and hook.sort_key < first_selected_sort_key:
+                expanded_hook_names.add(hook.name)
+                expanded_hook_names.add(Path(hook.name).stem)
+        expanded[plugin_name] = expanded_hook_names
+    return expanded
+
+
 def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]:
     from archivebox.config.common import get_config
     from archivebox.core.models import Snapshot
@@ -1459,8 +1471,8 @@ def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]
     crawl_plugin_names = [name.strip() for name in str((snapshot.crawl.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
     config_plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()]
     plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names or get_enabled_plugins(config=config)
-    catalog = _discover_archivebox_catalog()
-    plugins = catalog.select(plugin_names).plugins if plugin_names else catalog.plugins
+    catalog = get_plugin_catalog()
+    plugins = catalog.select(plugin_names) if plugin_names else catalog
     if snapshot.url == Snapshot.INTERNAL_INPUT_URL:
         plugins = {name: plugin for name, plugin in plugins.items() if getattr(plugin.config, "x_accepts_internal_input", False)}
     return sorted((plugin.name, hook.name) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot"))
@@ -1480,8 +1492,8 @@ def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None
     # The only thing queued work changes is the next scheduler value:
     # - open lifecycle rows: leave them due for extraction after maintenance
     # - no queued work left on a sealed row: clear retry_at
-    # - queued rows remain on a sealed Snapshot: leave it due so the search
-    #   backfill exception can process them on the next tick
+    # - queued rows remain on a sealed Snapshot: leave it due so targeted
+    #   plugin maintenance can process them on the next tick
     current_retry_at = snapshot.retry_at
     next_retry_at = timezone.now() if has_queued_results or snapshot.status in Snapshot.OPEN_STATES else None
     snapshot.retry_at = next_retry_at
@@ -1652,48 +1664,30 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
             snapshot.refresh_from_db()
         selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
         if selected_plugins:
-            search_only_plugins = all(plugin.startswith("search_backend_") for plugin in selected_plugins)
-            if not search_only_plugins:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                    current_step=0,
-                )
-                snapshot.refresh_from_db()
-            else:
-                _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-                run_crawl(
-                    str(snapshot.crawl_id),
-                    snapshot_ids=[str(snapshot.id)],
-                    selected_plugins=selected_plugins,
-                    process_discovered_snapshots_inline=True,
-                    interactive_interrupts=interactive_interrupts,
-                    config_overrides=config_overrides_for_queued_plugins(selected_plugins),
-                    selected_plugins_are_explicit=False,
-                )
-                from archivebox.core.models import ArchiveResult
+            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
+            run_crawl(
+                str(snapshot.crawl_id),
+                snapshot_ids=[str(snapshot.id)],
+                selected_plugins=selected_plugins,
+                process_discovered_snapshots_inline=True,
+                interactive_interrupts=interactive_interrupts,
+                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
+                selected_plugins_are_explicit=False,
+            )
+            from archivebox.core.models import ArchiveResult
 
-                has_queued_results = ArchiveResult.objects.filter(
-                    snapshot_id=snapshot.id,
-                    status=ArchiveResult.StatusChoices.QUEUED,
-                ).exists()
-                if not has_queued_results:
-                    type(snapshot).objects.filter(
-                        pk=snapshot.pk,
-                        status=snapshot.StatusChoices.SEALED,
-                    ).update(
-                        retry_at=None,
-                        modified_at=timezone.now(),
-                    )
-                else:
-                    type(snapshot).objects.filter(
-                        pk=snapshot.pk,
-                        status=snapshot.StatusChoices.SEALED,
-                    ).update(
-                        retry_at=timezone.now(),
-                        modified_at=timezone.now(),
-                    )
-                return True
+            has_queued_results = ArchiveResult.objects.filter(
+                snapshot_id=snapshot.id,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).exists()
+            type(snapshot).objects.filter(
+                pk=snapshot.pk,
+                status=snapshot.StatusChoices.SEALED,
+            ).update(
+                retry_at=timezone.now() if has_queued_results else None,
+                modified_at=timezone.now(),
+            )
+            return True
         if snapshot.status == Snapshot.StatusChoices.SEALED:
             if maintenance_ran:
                 return True
@@ -1709,10 +1703,10 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
         return False
     snapshot.refresh_from_db()
     if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        has_server_workset = snapshot.archiveresult_set.exclude(
+        has_server_archiveresults = snapshot.archiveresult_set.exclude(
             hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
         ).exists()
-        if has_server_workset and snapshot.is_finished_processing():
+        if has_server_archiveresults and snapshot.is_finished_processing():
             finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
             snapshot.refresh_from_db()
             if snapshot.status == Snapshot.StatusChoices.SEALED:
@@ -1721,10 +1715,10 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
                     snapshot.refresh_from_db()
                 _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
                 return True
-        if not has_server_workset:
-            # Materialize one durable row per configured plugin. Existing
-            # browser-uploaded rows are reused and queued so the server adds its
-            # outputs to that plugin result instead of creating a sibling row.
+        # A Snapshot with no server hook rows is fresh lifecycle work; materialize
+        # its configured hook set. Browser-extension uploads are completed external
+        # outputs, not the durable server workset, and must coexist with these rows.
+        if not has_server_archiveresults:
             snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
         snapshot.advance_lifecycle()
         snapshot.refresh_from_db()
@@ -1741,12 +1735,12 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
             return True
     if snapshot.status == Snapshot.StatusChoices.STARTED:
-        queued_plugins = queued_plugins_for_snapshot(str(snapshot.id))
-        if queued_plugins:
-            fail_unavailable_queued_plugins(
+        queued_plugins, selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(str(snapshot.id))
+        if queued_plugins and selected_hooks_by_plugin:
+            fail_unavailable_queued_hooks(
                 str(snapshot.id),
-                queued_plugins,
-                _discover_archivebox_plugins(),
+                selected_hooks_by_plugin,
+                get_plugin_catalog(),
             )
             if not queued_plugins_for_snapshot(str(snapshot.id)):
                 finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
@@ -1769,10 +1763,10 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
     )
     snapshot.refresh_from_db()
     if queued_plugins_for_snapshot(str(snapshot.id)):
-        # Plugin-level resume work is tracked by queued ArchiveResult rows, not by
+        # Hook-level resume work is tracked by queued ArchiveResult rows, not by
         # the Snapshot lease. If a partial pass returns with rows still queued,
         # wake the Snapshot immediately so takeover does not wait out a stale
-        # active-state lock before running the remaining plugins.
+        # active-state lock before running the remaining hooks.
         snapshot.update_and_requeue(retry_at=timezone.now())
     return True
 
@@ -1795,7 +1789,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     from archivebox.machine.models import Machine
     from archivebox.plugins.discovery import get_enabled_plugins
 
-    plugins = _discover_archivebox_plugins()
+    catalog = get_plugin_catalog()
     config = get_config(include_machine=False)
     machine = await sync_to_async(Machine.current, thread_sensitive=True)()
     derived_config = normalize_runtime_config(machine.config)
@@ -1808,18 +1802,28 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)
-    await _emit_machine_config(bus, config=config, derived_config=derived_config)
     live_stream = None
     bus_destroyed = False
 
     try:
         if plugin_names:
-            selected_plugins = filter_plugins(plugins, list(plugin_names), include_providers=True)
+            selected_plugins = catalog.select(plugin_names)
         else:
-            selected_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+            selected_plugins = catalog.select(get_enabled_plugins(config=config))
         if not selected_plugins:
             return
-        plugins_label = ", ".join(plugin_names) if plugin_names else f"enabled ({len(selected_plugins)} of {len(plugins)} available)"
+        plugins_label = ", ".join(plugin_names) if plugin_names else f"enabled ({len(selected_plugins)} of {len(catalog)} available)"
+        install_config = dict(config)
+        for plugin in selected_plugins.values():
+            if plugin.enabled_key in plugin.config.properties:
+                install_config[plugin.enabled_key] = True
+        plan = ExecutionPlan.build(
+            catalog,
+            selected_plugins=list(selected_plugins),
+            config=install_config,
+            derived_config=derived_config,
+            runtime="archivebox",
+        )
         timeout_seconds = config["TIMEOUT"]
         stdout_is_tty = sys.stdout.isatty()
         stderr_is_tty = sys.stderr.isatty()
@@ -1872,11 +1876,8 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
             with live_ui if live_ui is not None else nullcontext():
                 try:
                     await abx_install_plugins(
-                        plugin_names=selected_plugins,
-                        plugins=plugins,
+                        plan,
                         output_dir=output_dir,
-                        config_overrides=config,
-                        derived_config_overrides=derived_config,
                         emit_jsonl=False,
                         bus=bus,
                         BinaryService=None,
@@ -2047,23 +2048,22 @@ def _run_due_queued_plugin_result(
         config_overrides=config_overrides_for_queued_plugins(selected_plugins, CRAWL_MAX_CONCURRENT_SNAPSHOTS=batch_size),
         selected_plugins_are_explicit=False,
     )
-    if all(plugin.startswith("search_backend_") for plugin in selected_plugins):
-        queued_results = ArchiveResult.objects.filter(
-            snapshot_id=OuterRef("pk"),
-            status=ArchiveResult.StatusChoices.QUEUED,
-            plugin__in=selected_plugins,
-        )
-        Snapshot.objects.filter(
-            id__in=claimed_snapshot_ids,
-            status=Snapshot.StatusChoices.SEALED,
-        ).annotate(
-            has_queued_results=Exists(queued_results),
-        ).filter(
-            has_queued_results=False,
-        ).update(
-            retry_at=None,
-            modified_at=timezone.now(),
-        )
+    queued_results = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
+        status=ArchiveResult.StatusChoices.QUEUED,
+        plugin__in=selected_plugins,
+    )
+    Snapshot.objects.filter(
+        id__in=claimed_snapshot_ids,
+        status=Snapshot.StatusChoices.SEALED,
+    ).annotate(
+        has_queued_results=Exists(queued_results),
+    ).filter(
+        has_queued_results=False,
+    ).update(
+        retry_at=None,
+        modified_at=timezone.now(),
+    )
     return True
 
 
@@ -2229,21 +2229,17 @@ def run_pending_crawls(
             ):
                 continue
 
-        # Search backend selection is live crawl-execution config, not an
-        # installed-plugin list. Old queued rows for a backend that is disabled
-        # by the current Machine/Crawl/Snapshot config must remain queued so
-        # they can run if the user re-enables that backend, but they should not
-        # launch a standalone hook process just to skip after imports/config
-        # hydration. Refreshing here preserves mid-run config edits while using
-        # the same enabled-hook discovery path that created ArchiveResult rows.
+        # Plugin selection is live crawl-execution config, not an installed-
+        # plugin list. Old queued rows for a plugin that is disabled by the
+        # current Machine/Crawl/Snapshot config remain queued until the user
+        # re-enables it. Refresh here to preserve mid-run config edits while
+        # using the same enabled-hook discovery path that created the rows.
         runtime_config = get_config()
         catalog = get_plugin_catalog()
         enabled_plugins = get_enabled_plugins(config=runtime_config)
-        search_plugin_names = frozenset(
-            plugin.name for plugin, _hook in catalog.hooks("Snapshot", names=enabled_plugins) if plugin.name.startswith("search_backend_")
-        )
+        queued_plugin_names = frozenset(plugin.name for plugin, _hook in catalog.hooks("Snapshot", names=enabled_plugins))
         if _run_due_queued_plugin_result(
-            search_plugin_names,
+            queued_plugin_names,
             crawl_id=crawl_id,
             lock_seconds=60,
             interactive_interrupts=interactive_interrupts,
@@ -2261,13 +2257,13 @@ def run_pending_crawls(
                 retry_at__lte=timezone.now(),
                 status=Snapshot.StatusChoices.SEALED,
             )
-            if search_plugin_names:
-                queued_search_snapshot_ids = ArchiveResult.objects.filter(
+            if queued_plugin_names:
+                queued_plugin_snapshot_ids = ArchiveResult.objects.filter(
                     status=ArchiveResult.StatusChoices.QUEUED,
-                    plugin__in=search_plugin_names,
+                    plugin__in=queued_plugin_names,
                 ).values("snapshot_id")
                 sealed_snapshots = sealed_snapshots.exclude(
-                    id__in=queued_search_snapshot_ids,
+                    id__in=queued_plugin_snapshot_ids,
                 )
             if crawl_id:
                 sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)

archivebox/services/tag_service.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from asgiref.sync import sync_to_async
+
 from abx_dl.events import TagEvent
 from abx_dl.services.base import BaseService
 
@@ -13,10 +15,10 @@ class TagService(BaseService):
         self.bus.on(TagEvent, self.on_TagEvent__save_to_db)
 
     async def on_TagEvent__save_to_db(self, event: TagEvent) -> None:
-        from archivebox.core.models import Snapshot, SnapshotTag, Tag
+        from archivebox.core.models import Snapshot, Tag
 
         snapshot = await Snapshot.objects.filter(id=event.snapshot_id).afirst()
         if snapshot is None:
             return
-        tag, _ = await Tag.objects.aget_or_create(name=event.name)
-        await SnapshotTag.objects.aget_or_create(snapshot=snapshot, tag=tag)
+        tag, _ = await sync_to_async(Tag.get_or_create_by_name, thread_sensitive=True)(event.name)
+        await sync_to_async(snapshot.add_tag_ids, thread_sensitive=True)([tag.pk])

archivebox/tests/conftest.py

@@ -1660,6 +1660,74 @@ def resolve_abxpkg_binary_env(
     return {str(key): str(value) for key, value in payload.items()}
 
 
+def run_test_hook(
+    script: Path,
+    output_dir: Path,
+    config: dict[str, Any] | None = None,
+    timeout: int = 60,
+    **arguments: Any,
+):
+    """Execute a shipped finite hook through abx-dl and ArchiveBox's real DB projector."""
+    import asyncio
+
+    from abx_dl.execution import execute_hook
+    from abx_dl.orchestrator import create_bus
+    from archivebox.machine.models import Process
+    from archivebox.plugins.discovery import get_plugin_catalog
+    from archivebox.services.process_service import ProcessService, parse_event_datetime
+
+    resolved_script = script.resolve()
+    hook = next(
+        (hook for plugin in get_plugin_catalog().values() for hook in plugin.hooks if hook.path.resolve() == resolved_script),
+        None,
+    )
+    assert hook is not None, f"shipped hook is not in the plugin catalog: {script}"
+    assert not hook.is_background, f"run_test_hook only supports finite hooks: {hook.full_name}"
+
+    env = os.environ.copy()
+    for key, value in (config or {}).items():
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            env[key] = "true" if value else "false"
+        elif isinstance(value, (dict, list, tuple)):
+            env[key] = json.dumps(value)
+        else:
+            env[key] = str(value)
+    if env.get("NODE_MODULES_DIR"):
+        env.setdefault("NODE_MODULE_DIR", env["NODE_MODULES_DIR"])
+
+    bus = create_bus(name=f"test_hook_{hook.plugin_name}", total_timeout=float(timeout) + 30.0)
+    ProcessService(bus)
+
+    async def execute_and_close():
+        try:
+            return await execute_hook(
+                hook,
+                output_dir=output_dir,
+                env=env,
+                arguments=arguments,
+                timeout=timeout,
+                bus=bus,
+                process_type=Process.TypeChoices.HOOK,
+            )
+        finally:
+            await bus.wait_until_idle()
+            await bus.destroy(clear=False)
+
+    completed = asyncio.run(execute_and_close())
+    process = (
+        Process.objects.filter(
+            pid=completed.pid or None,
+            started_at=parse_event_datetime(completed.start_ts),
+        )
+        .order_by("-modified_at")
+        .first()
+    )
+    assert process is not None, f"hook completed without an ArchiveBox Process projection: {hook.full_name}"
+    return process
+
+
 def resolve_abxpkg_chrome_env(lib_dir: Path, env: dict[str, str] | None = None) -> dict[str, str]:
     from abx_plugins import get_plugins_dir
 

archivebox/tests/test_api_v1_core_archiveresults.py

@@ -3,6 +3,7 @@ from datetime import timedelta
 import pytest
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import connection
+from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart
 from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 
@@ -14,7 +15,7 @@ from archivebox.tests.conftest import api_client_request
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_archiveresult_upload_upserts_one_snapshot_plugin_result(client, api_admin_user, api_headers):
+def test_archiveresult_upload_upserts_by_snapshot_plugin_and_hook(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
     snapshot = Snapshot.objects.create(url="https://example.com/unified-result", crawl=crawl)
 
@@ -39,13 +40,130 @@ def test_archiveresult_upload_upserts_one_snapshot_plugin_result(client, api_adm
     assert server_response.status_code == 200, server_response.content
     assert extension_update.status_code == 200, extension_update.content
     assert extension_update.json()["id"] == extension_response.json()["id"]
-    assert server_response.json()["id"] == extension_response.json()["id"]
+    assert server_response.json()["id"] != extension_response.json()["id"]
 
-    results = ArchiveResult.objects.filter(snapshot=snapshot, plugin="screenshot")
-    assert results.count() == 1
-    result = results.get()
-    assert result.hook_name == "on_Snapshot__archivebox_browser_extension_upload"
-    assert set(result.output_files) == {"browser.png", "server.png", "browser-2.png"}
+    results = ArchiveResult.objects.filter(snapshot=snapshot, plugin="screenshot").order_by("hook_name")
+    assert results.count() == 2
+    extension_result = results.get(hook_name="on_Snapshot__archivebox_browser_extension_upload")
+    server_result = results.get(hook_name="on_Snapshot__50_screenshot")
+    assert set(extension_result.output_files) == {"browser.png", "browser-2.png"}
+    assert set(server_result.output_files) == {"server.png"}
+    snapshot.refresh_from_db()
+    assert snapshot.output_size == len(b"browser") + len(b"browser-2") + len(b"server")
+
+
+def test_archiveresult_create_does_not_open_a_database_transaction(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/autocommit-result", crawl=crawl)
+
+    with CaptureQueriesContext(connection) as queries:
+        response = client.post(
+            "/api/v1/core/archiveresults",
+            {
+                "snapshot_id": str(snapshot.id),
+                "plugin": "chrome_extension_screenshot",
+                "files": SimpleUploadedFile("screenshot.png", b"screenshot", content_type="image/png"),
+                "output_paths": "screenshot.png",
+            },
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_intermediate_archiveresult_chunks_only_write_to_disk(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/chunked-result", crawl=crawl)
+    create_response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": str(snapshot.id),
+            "plugin": "chrome_extension_mhtml",
+            "status": ArchiveResult.StatusChoices.STARTED,
+        },
+        **api_headers,
+    )
+    assert create_response.status_code == 200, create_response.content
+    result = ArchiveResult.objects.get(pk=create_response.json()["id"])
+    original_modified_at = result.modified_at
+
+    with CaptureQueriesContext(connection) as queries:
+        chunk_response = client.patch(
+            f"/api/v1/core/archiveresult/{result.id}",
+            encode_multipart(
+                BOUNDARY,
+                {
+                    "files": SimpleUploadedFile("snapshot.mhtml.part-000000", b"first", content_type="multipart/related"),
+                    "chunk_output_path": "snapshot.mhtml",
+                    "chunk_index": "0",
+                    "chunk_count": "2",
+                    "chunk_offset": "0",
+                    "chunk_total_size": "11",
+                    "mime_type": "multipart/related",
+                    "status": ArchiveResult.StatusChoices.STARTED,
+                },
+            ),
+            content_type=MULTIPART_CONTENT,
+            **api_headers,
+        )
+
+    assert chunk_response.status_code == 200, chunk_response.content
+    result.refresh_from_db()
+    assert result.output_files == {}
+    assert result.output_size == 0
+    assert result.modified_at == original_modified_at
+    writes = [query["sql"] for query in queries if query["sql"].lstrip().upper().startswith(("INSERT", "UPDATE", "DELETE"))]
+    assert writes == []
+
+    final_response = client.patch(
+        f"/api/v1/core/archiveresult/{result.id}",
+        encode_multipart(
+            BOUNDARY,
+            {
+                "files": SimpleUploadedFile("snapshot.mhtml.part-000001", b"-final", content_type="multipart/related"),
+                "chunk_output_path": "snapshot.mhtml",
+                "chunk_index": "1",
+                "chunk_count": "2",
+                "chunk_offset": "5",
+                "chunk_total_size": "11",
+                "mime_type": "multipart/related",
+                "status": ArchiveResult.StatusChoices.SUCCEEDED,
+            },
+        ),
+        content_type=MULTIPART_CONTENT,
+        **api_headers,
+    )
+    assert final_response.status_code == 200, final_response.content
+    result.refresh_from_db()
+    assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert result.output_size == 11
+    assert result.output_files["snapshot.mhtml"]["upload"]["complete"] is True
+    snapshot.refresh_from_db()
+    assert snapshot.output_size == 11
+
+
+def test_archiveresult_safe_update_rejects_stale_writers(api_admin_user):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/cas-result", crawl=crawl)
+    result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="chrome_extension_dom",
+        hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
+        status=ArchiveResult.StatusChoices.STARTED,
+        output_size=1,
+    )
+    stale_result = ArchiveResult.objects.get(pk=result.pk)
+
+    assert result.safe_update({"output_size": 2}) is True
+    assert stale_result.safe_update({"output_size": 3}) is False
+
+    stale_result.refresh_from_db()
+    snapshot.refresh_from_db()
+    assert stale_result.output_size == 2
+    assert snapshot.output_size == 2
 
 
 def test_archiveresult_upload_api_queues_snapshot_maintenance_without_finalizing(client, api_admin_user, api_headers):

archivebox/tests/test_api_v1_core_snapshots.py

@@ -2,6 +2,8 @@ import threading
 import time
 
 import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot
 from archivebox.crawls.locks import crawl_lifecycle_lock
@@ -85,3 +87,66 @@ def test_existing_snapshot_metadata_sync_does_not_wait_for_active_crawl(client,
     assert response.status_code == 200, response.content
     assert response.json()["id"] == str(snapshot.id)
     assert elapsed < 1
+
+
+def test_new_snapshot_creation_does_not_open_a_database_transaction(client, api_admin_user, api_headers):
+    url = "https://example.com/browser-extension-new-snapshot"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+
+    with CaptureQueriesContext(connection) as queries:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={
+                "url": url,
+                "crawl_id": str(crawl.id),
+                "depth": 0,
+                "status": Snapshot.StatusChoices.QUEUED,
+                "tags": ["browser-extension-upload"],
+            },
+            content_type="application/json",
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
+    assert Snapshot.objects.get(url=url, crawl=crawl).tags.filter(name="browser-extension-upload").exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_user, api_headers):
+    url = "https://example.com/browser-extension-new-snapshot-active-crawl"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    lock_acquired = threading.Event()
+    release_lock = threading.Event()
+
+    def hold_active_crawl_lock():
+        with crawl_lifecycle_lock(str(crawl.id)):
+            lock_acquired.set()
+            release_lock.wait(timeout=3)
+
+    holder = threading.Thread(target=hold_active_crawl_lock)
+    holder.start()
+    assert lock_acquired.wait(timeout=1)
+
+    started_at = time.monotonic()
+    response = client.post(
+        "/api/v1/core/snapshots",
+        data={
+            "url": url,
+            "crawl_id": str(crawl.id),
+            "depth": 0,
+            "status": Snapshot.StatusChoices.QUEUED,
+            "tags": ["browser-extension-upload"],
+        },
+        content_type="application/json",
+        **api_headers,
+    )
+    elapsed = time.monotonic() - started_at
+    release_lock.set()
+    holder.join(timeout=3)
+
+    assert response.status_code == 200, response.content
+    assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
+    assert elapsed < 1

archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py

@@ -1,4 +1,6 @@
 import pytest
+from django.db import IntegrityError, connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot, Tag
 from archivebox.crawls.models import Crawl
@@ -13,13 +15,39 @@ def test_basic_success_case_request(client, api_admin_user, api_headers):
     snapshot = Snapshot.objects.create(url="https://example.com/tag-add", crawl=crawl)
     tag = Tag.objects.create(name="api-basic-add-tag", created_by=api_admin_user)
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/tags/add-to-snapshot/",
-        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
-        headers=api_headers,
-    )
+    with CaptureQueriesContext(connection) as queries:
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/tags/add-to-snapshot/",
+            payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+            headers=api_headers,
+        )
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+    assert snapshot.tags.filter(pk=tag.pk).exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_add_tag_ids_reraises_non_duplicate_integrity_errors(snapshot):
+    with pytest.raises(IntegrityError):
+        snapshot.add_tag_ids([2**31 - 1])
+
+
+def test_add_tag_ids_treats_existing_snapshot_tag_as_idempotent(snapshot, admin_user):
+    tag = Tag.objects.create(name="already-attached", created_by=admin_user)
+
+    snapshot.add_tag_ids([tag.pk])
+    snapshot.add_tag_ids([tag.pk])
+
+    assert snapshot.tags.filter(pk=tag.pk).count() == 1
+
+
+def test_save_tags_without_creator_uses_tag_model_default(snapshot):
+    snapshot.save_tags(["model-default-creator"])
+
+    tag = Tag.objects.get(name="model-default-creator")
+    assert tag.created_by_id is not None

archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py

@@ -1,4 +1,6 @@
 import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot, Tag
 from archivebox.crawls.models import Crawl
@@ -14,13 +16,18 @@ def test_basic_success_case_request(client, api_admin_user, api_headers):
     tag = Tag.objects.create(name="api-basic-remove-tag", created_by=api_admin_user)
     snapshot.tags.add(tag)
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/tags/remove-from-snapshot/",
-        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
-        headers=api_headers,
-    )
+    with CaptureQueriesContext(connection) as queries:
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/tags/remove-from-snapshot/",
+            payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+            headers=api_headers,
+        )
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+    assert not snapshot.tags.filter(pk=tag.pk).exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []

archivebox/tests/test_archive_result_service.py

@@ -19,9 +19,9 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def _snapshot_hook_name(plugin_name: str) -> str:
-    from abx_dl.models import discover_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
-    plugin = discover_plugins().get(plugin_name)
+    plugin = get_plugin_catalog().get(plugin_name)
     assert plugin is not None, f"missing test plugin {plugin_name}"
     hooks = plugin.filter_hooks("Snapshot")
     assert hooks, f"missing Snapshot hooks for {plugin_name}"
@@ -47,15 +47,15 @@ def _run_shipped_snapshot_hook(
     """Run one shipped hook through the production process/result bus services."""
     import asyncio
 
-    from abx_dl.models import discover_plugins
     from abx_dl.services.process_service import ProcessService as HookProcessService
     from abx_plugins.plugins.base.utils import get_hydrated_required_binaries
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
+    from archivebox.plugins.discovery import get_plugin_catalog
     from archivebox.services.archive_result_service import ArchiveResultService
     from archivebox.services.process_service import ProcessService as PersistedProcessService
 
-    discovered_plugin = discover_plugins().get(plugin)
+    discovered_plugin = get_plugin_catalog().get(plugin)
     assert discovered_plugin is not None, f"missing test plugin {plugin}"
     matching_hooks = [hook for hook in discovered_plugin.filter_hooks("Snapshot") if hook.name == hook_name or hook.path.name == hook_name]
     assert len(matching_hooks) == 1, f"missing or ambiguous Snapshot hook {plugin}:{hook_name}"
@@ -230,7 +230,7 @@ def test_archiveresult_event_retry_updates_existing_hook_row(tmp_path, hermetic_
     _cleanup_machine_process_rows()
 
 
-def test_archiveresult_duplicate_plugin_rows_are_rejected():
+def test_archiveresult_duplicate_hook_rows_are_rejected():
     from django.db import IntegrityError, transaction
     from archivebox.core.models import ArchiveResult
 
@@ -246,45 +246,35 @@ def test_archiveresult_duplicate_plugin_rows_are_rejected():
         ArchiveResult.objects.create(
             snapshot=snapshot,
             plugin="wget",
-            hook_name="on_Snapshot__99_other_wget_hook",
+            hook_name="on_Snapshot__06_wget.finite.bg",
             status=ArchiveResult.StatusChoices.SUCCEEDED,
         )
 
 
-def test_archivewebpage_lifecycle_hooks_project_one_plugin_output():
+def test_archiveresult_event_create_uses_one_result_lookup():
     from abx_dl.events import ArchiveResultEvent
-    from archivebox.core.models import ArchiveResult
+    from django.db import connection
+    from django.test.utils import CaptureQueriesContext
+
     from archivebox.services.archive_result_service import _save_archiveresult_event_to_db
 
     snapshot = _create_snapshot()
-    _save_archiveresult_event_to_db(
-        ArchiveResultEvent(
-            snapshot_id=str(snapshot.id),
-            plugin="archivewebpage",
-            hook_name="on_Snapshot__16_archivewebpage_start",
-            status="succeeded",
-            output_str="recording started",
-            output_files=[OutputFile(path="recording.json", extension="json", mimetype="application/json", size=175)],
-        ),
-        None,
-    )
-    _save_archiveresult_event_to_db(
-        ArchiveResultEvent(
-            snapshot_id=str(snapshot.id),
-            plugin="archivewebpage",
-            hook_name="on_Snapshot__65_archivewebpage_stop",
-            status="succeeded",
-            output_str="archivewebpage.wacz",
-            output_files=[OutputFile(path="archivewebpage.wacz", extension="wacz", size=2048)],
-        ),
-        None,
+    event = ArchiveResultEvent(
+        snapshot_id=str(snapshot.id),
+        plugin="review-query-count",
+        hook_name="on_Snapshot__99_review.py",
+        status="failed",
     )
 
-    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="archivewebpage")
-    assert result.hook_name == "on_Snapshot__65_archivewebpage_stop"
-    assert result.output_str == "archivewebpage.wacz"
-    assert set(result.output_files) == {"recording.json", "archivewebpage.wacz"}
-    assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="archivewebpage").count() == 1
+    with CaptureQueriesContext(connection) as queries:
+        _save_archiveresult_event_to_db(event, None)
+
+    result_lookups = [
+        query["sql"]
+        for query in queries
+        if query["sql"].lstrip().upper().startswith("SELECT") and 'FROM "core_archiveresult"' in query["sql"]
+    ]
+    assert len(result_lookups) == 1
 
 
 def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_path, hermetic_lib_dir):
@@ -470,13 +460,12 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
     reset_count = snapshot.retry_failed_archiveresults()
 
     snapshot.refresh_from_db()
-    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="chrome")
+    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="chrome", hook_name="on_Snapshot__11_chrome_wait")
     assert reset_count == 1
     assert snapshot.status == Snapshot.StatusChoices.QUEUED
     assert snapshot.retry_at is not None
     assert snapshot.current_step == 0
     assert result.status == ArchiveResult.StatusChoices.QUEUED
-    assert result.hook_name == ""
     assert result.output_str == ""
     assert result.output_json is None
     assert result.output_files == {}

archivebox/tests/test_cli_archiveresult.py

@@ -61,11 +61,10 @@ class TestArchiveResultCreate:
 
         ar = next(r for r in records if r["type"] == "ArchiveResult")
         assert ar["plugin"] == "title"
-        # Queue projection is one row per plugin, while a plugin can contain
-        # several ordered hooks. The runner records the concrete hook only
-        # after execution; inventing one here would make the pending row claim
-        # work that has not run and reintroduce hook-level duplicate results.
-        assert ar["hook_name"] == ""
+        # Hook identity is known at scheduling time and is the durable retry
+        # key. Emitting it here prevents multiple hooks in one plugin from
+        # collapsing into the same ArchiveResult row.
+        assert ar["hook_name"] == "on_Snapshot__54_title"
         assert "id" not in ar
 
     def test_create_with_specific_plugin(self, initialized_archive):
@@ -92,11 +91,9 @@ class TestArchiveResultCreate:
         assert code == 0
         records = parse_jsonl_output(stdout2)
         ar_records = [r for r in records if r.get("type") == "ArchiveResult"]
-        assert len(ar_records) >= 1
+        assert len(ar_records) == 1
         assert all(record["plugin"] == "screenshot" for record in ar_records)
-        # A requested plugin is the schedulable unit; its concrete hook is an
-        # execution result, not input metadata on this pre-execution request.
-        assert all(record["hook_name"] == "" for record in ar_records)
+        assert [record["hook_name"] for record in ar_records] == ["on_Snapshot__51_screenshot"]
 
     def test_create_pass_through_crawl(self, initialized_archive):
         """Pass-through Crawl records unchanged."""

archivebox/tests/test_cli_extract_input.py

@@ -22,7 +22,7 @@ def create_extract_snapshot(initialized_archive, env, url="https://example.com")
     )
 
 
-def test_extract_archiveresult_record_queues_one_plugin_result(initialized_archive):
+def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive):
     env = cli_env(PLUGINS="archivewebpage")
     create_extract_snapshot(initialized_archive, env)
 
@@ -52,7 +52,7 @@ def test_extract_archiveresult_record_queues_one_plugin_result(initialized_archi
         rows = list(
             ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage").values_list("hook_name", "status"),
         )
-    assert rows == [("", ArchiveResult.StatusChoices.QUEUED)]
+    assert rows == [(hook_name, ArchiveResult.StatusChoices.QUEUED)]
 
 
 def test_extract_runs_on_snapshot_id(initialized_archive):

archivebox/tests/test_cli_piping.py

@@ -38,7 +38,7 @@ PIPE_TEST_ENV = {
 
 def run_real_txt_parser(tmp_path, text):
     """Run the shipped text parser and return its real snapshot output directory."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "parser-snapshot"
     staticfile_dir = snap_dir / "staticfile"
@@ -47,7 +47,7 @@ def run_real_txt_parser(tmp_path, text):
     output_dir.mkdir(parents=True)
     (staticfile_dir / "input.txt").write_text(text, encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_cli_run.py

@@ -240,7 +240,7 @@ class TestRunWithArchiveResult:
     """Tests for `archivebox run` with ArchiveResult input."""
 
     @pytest.mark.django_db(transaction=True)
-    def test_run_treats_no_id_archiveresult_request_as_plugin_work(self, initialized_archive):
+    def test_run_creates_and_runs_exact_no_id_archiveresult_request(self, initialized_archive):
         import json
 
         from archivebox.core.models import ArchiveResult
@@ -284,9 +284,13 @@ class TestRunWithArchiveResult:
                     "output_str",
                 ),
             )
-        assert len(rows) == 1
-        assert rows[0][0] != missing_hook
-        assert rows[0][1] in ArchiveResult.FINAL_STATES
+        assert rows == [
+            (
+                missing_hook,
+                ArchiveResult.StatusChoices.FAILED,
+                "Queued hook is no longer available in the installed plugin",
+            ),
+        ]
 
     def test_run_requeues_failed_archiveresult(self, initialized_archive):
         """Run re-queues a failed ArchiveResult."""
@@ -1072,7 +1076,7 @@ class TestRecoverOrchestratorState:
         assert snapshot.retry_at is None
         assert snapshot.downloaded_at is not None
 
-    def test_create_pending_archiveresults_creates_one_plugin_row_not_hook_rows(self):
+    def test_create_pending_archiveresults_uses_canonical_hook_names(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -1092,17 +1096,11 @@ class TestRecoverOrchestratorState:
             retry_at=timezone.now(),
         )
 
-        results = snapshot.create_pending_archiveresults(
-            hooks=[
-                ("chrome", "on_Snapshot__00_chrome_launch.daemon.bg"),
-                ("chrome", "on_Snapshot__01_chrome_tab.daemon.bg"),
-                ("chrome", "on_Snapshot__30_chrome_navigate"),
-                ("title", "on_Snapshot__54_title"),
-            ],
-        )
+        snapshot.create_pending_archiveresults()
 
-        assert [(result.plugin, result.hook_name) for result in results] == [("chrome", ""), ("title", "")]
-        assert ArchiveResult.objects.filter(snapshot=snapshot).count() == 2
+        hook_names = list(ArchiveResult.objects.filter(snapshot=snapshot).values_list("hook_name", flat=True))
+        assert hook_names
+        assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
 
     def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self):
         from django.utils import timezone
@@ -1697,13 +1695,11 @@ class TestRecoverOrchestratorState:
         Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
         snapshot.refresh_from_db()
         snapshot.output_dir.mkdir(parents=True, exist_ok=True)
-        title_dir = snapshot.output_dir / "title"
-        title_dir.mkdir(parents=True, exist_ok=True)
-        (title_dir / "title.txt").write_text("Example Domain\n", encoding="utf-8")
+        (snapshot.output_dir / "source.txt").write_text("real targeted maintenance input\n", encoding="utf-8")
         result = ArchiveResult.objects.create(
             snapshot=snapshot,
-            plugin="search_backend_sqlite",
-            hook_name="on_Snapshot__90_index_sqlite",
+            plugin="hashes",
+            hook_name="on_Snapshot__93_hashes.py",
             status=ArchiveResult.StatusChoices.QUEUED,
         )
 
@@ -1718,7 +1714,7 @@ class TestRecoverOrchestratorState:
         assert result.end_ts is not None
 
     @pytest.mark.django_db(transaction=True)
-    def test_run_due_snapshot_runs_plugin_for_obsolete_queued_hook_name(self):
+    def test_run_due_snapshot_fails_obsolete_queued_hook_name(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -1749,8 +1745,8 @@ class TestRecoverOrchestratorState:
 
         result.refresh_from_db()
         snapshot.refresh_from_db()
-        assert result.status in ArchiveResult.FINAL_STATES
-        assert result.hook_name != "on_Snapshot__50_singlefile.py"
+        assert result.status == ArchiveResult.StatusChoices.FAILED
+        assert result.output_str == "Queued hook is no longer available in the installed plugin"
         assert snapshot.retry_at is None
 
     @pytest.mark.django_db(transaction=True)
@@ -1829,10 +1825,7 @@ class TestRecoverOrchestratorState:
         recursive_test_site,
         chrome_isolation,
     ):
-        from pathlib import Path
-
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.machine.models import Process
+        from archivebox.core.models import ArchiveResult
         from archivebox.tests.test_orm_helpers import use_archivebox_db
 
         env = cli_env(disable_extractors=True)
@@ -1870,19 +1863,13 @@ class TestRecoverOrchestratorState:
         snapshot_id = navigate_record["snapshot_id"]
 
         with use_archivebox_db(initialized_archive):
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            tab_processes = [
-                process
-                for process in Process.objects.filter(pwd=str(Path(snapshot.output_dir) / "chrome")).order_by("-started_at")
-                if Path(process.hook_script_name or "").stem == "on_Snapshot__01_chrome_tab.daemon.bg"
-            ]
-            assert tab_processes
-            first_tab_process_id = tab_processes[0].id
-            plugin_result = ArchiveResult.objects.get(
+            tab_result = ArchiveResult.objects.get(
                 snapshot_id=snapshot_id,
                 plugin="chrome",
+                hook_name="on_Snapshot__01_chrome_tab.daemon.bg",
             )
-            assert plugin_result.hook_name == "on_Snapshot__30_chrome_navigate"
+            first_tab_process_id = tab_result.process_id
+            assert first_tab_process_id is not None
 
         update_process = run_archivebox_cmd(
             ["archiveresult", "update", "--status=queued"],
@@ -1916,18 +1903,18 @@ class TestRecoverOrchestratorState:
             navigate_result = ArchiveResult.objects.get(
                 snapshot_id=snapshot_id,
                 plugin="chrome",
+                hook_name="on_Snapshot__30_chrome_navigate",
+            )
+            tab_result = ArchiveResult.objects.get(
+                snapshot_id=snapshot_id,
+                plugin="chrome",
+                hook_name="on_Snapshot__01_chrome_tab.daemon.bg",
             )
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            tab_processes = [
-                process
-                for process in Process.objects.filter(pwd=str(Path(snapshot.output_dir) / "chrome")).order_by("-started_at")
-                if Path(process.hook_script_name or "").stem == "on_Snapshot__01_chrome_tab.daemon.bg"
-            ]
 
         assert run_process.returncode == 0
         assert navigate_result.status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert navigate_result.hook_name == "on_Snapshot__30_chrome_navigate"
-        assert tab_processes[0].id != first_tab_process_id
+        assert tab_result.process_id is not None
+        assert tab_result.process_id != first_tab_process_id
 
     def test_recover_orchestrator_state_ignores_sealed_downloaded_snapshot_without_results(self):
         from django.utils import timezone
@@ -2186,6 +2173,7 @@ class TestRunDueCrawlState:
         assert finished.output_str == "keep me"
         assert finished.output_files == {"favicon.ico": {"size": 1}}
 
+    @pytest.mark.django_db(transaction=True)
     def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self):
         from importlib.resources import files
         from pathlib import Path
@@ -2195,7 +2183,7 @@ class TestRunDueCrawlState:
         from archivebox.base_models.models import get_or_create_system_user_pk
         from archivebox.core.models import ArchiveResult, Snapshot
         from archivebox.crawls.models import Crawl
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
+        from archivebox.tests.conftest import run_test_hook
         from archivebox.services.runner import run_due_snapshot
 
         crawl = Crawl.objects.create(
@@ -2221,7 +2209,7 @@ class TestRunDueCrawlState:
             encoding="utf-8",
         )
         hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             parser_dir,
             config={"ABXPKG_LIB_DIR": str(root.output_dir.parent.parent / "lib"), "SNAP_DIR": str(root.output_dir)},
@@ -2232,12 +2220,12 @@ class TestRunDueCrawlState:
         )
         process.refresh_from_db()
         assert process.exit_code == 0, process.stderr
-        result_record = next(record for record in extract_records_from_process(process) if record.get("type") == "ArchiveResult")
+        result_record = next(record for record in process.get_records() if record.get("type") == "ArchiveResult")
         ArchiveResult.objects.create(
             snapshot=root,
             process=process,
-            plugin=result_record["plugin"],
-            hook_name=result_record["hook_name"],
+            plugin="parse_txt_urls",
+            hook_name=hook_path.name,
             status=result_record["status"],
             output_str=result_record.get("output_str", ""),
             output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}},

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -15,7 +15,7 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_only_sealed_search_backfill_bypasses_snapshot_lifecycle():
+def test_targeted_plugin_retries_preserve_sealed_snapshot_lifecycle():
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.cli.archivebox_extract import run_plugins
     from archivebox.core.models import ArchiveResult
@@ -85,8 +85,8 @@ def test_only_sealed_search_backfill_bypasses_snapshot_lifecycle():
     )
     extract_crawl.refresh_from_db()
     extract_snapshot.refresh_from_db()
-    assert extract_crawl.status == Crawl.StatusChoices.QUEUED
-    assert extract_snapshot.status == Snapshot.StatusChoices.QUEUED
+    assert extract_crawl.status == Crawl.StatusChoices.SEALED
+    assert extract_snapshot.status == Snapshot.StatusChoices.SEALED
     assert extract_snapshot.archiveresult_set.filter(
         plugin="wget",
         status=ArchiveResult.StatusChoices.QUEUED,

archivebox/tests/test_config_MAX_limits.py

@@ -67,7 +67,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import CrawlRunner
 
     crawl = Crawl.objects.create(
@@ -92,7 +92,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
         encoding="utf-8",
     )
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         parser_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_crawl_runner.py

@@ -657,6 +657,35 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp
     assert snapshot.archiveresult_set.count() == 0
 
 
+@pytest.mark.django_db(transaction=True)
+def test_crawl_runner_preflights_plugins_before_crawl_lifecycle(tmp_path):
+    from abx_dl.events import CrawlEvent, CrawlSetupEvent, InstallEvent
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.services.runner import CrawlRunner
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        config={
+            "ABXPKG_LIB_DIR": str(tmp_path / "lib"),
+            "PLUGINS": "base",
+            "CHROME_BINARY": "",
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl)
+    event_order = []
+    runner.bus.on(InstallEvent, lambda _event: event_order.append(InstallEvent))
+    runner.bus.on(CrawlEvent, lambda _event: event_order.append(CrawlEvent))
+    runner.bus.on(CrawlSetupEvent, lambda _event: event_order.append(CrawlSetupEvent))
+
+    asyncio.run(runner.run())
+
+    assert event_order.count(InstallEvent) == 1
+    assert event_order.index(InstallEvent) < event_order.index(CrawlEvent)
+    assert event_order.index(InstallEvent) < event_order.index(CrawlSetupEvent)
+
+
 @pytest.mark.django_db(transaction=True)
 def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot():
     from abx_dl.events import SnapshotCompletedEvent

archivebox/tests/test_hooks.py

@@ -80,14 +80,14 @@ class TestJSONLParsing:
 
     @staticmethod
     def run_hashes_hook(tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "hash-snapshot"
         output_dir = snap_dir / "hashes"
         output_dir.mkdir(parents=True)
         (snap_dir / "source.txt").write_text("real parser input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -100,7 +100,7 @@ class TestJSONLParsing:
 
     @staticmethod
     def run_parser_hook(tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "parser-snapshot"
         staticfile_dir = snap_dir / "staticfile"
@@ -112,7 +112,7 @@ class TestJSONLParsing:
             encoding="utf-8",
         )
         hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -182,35 +182,6 @@ class TestJSONLParsing:
         assert len(records) == 1
         assert records[0]["type"] == "ArchiveResult"
 
-    def test_direct_hook_preserves_explicit_parent_process(self, tmp_path):
-        """The compatibility adapter must retain the caller's process hierarchy."""
-        from archivebox.machine.models import Machine, Process
-        from archivebox.plugins.hooks import run_hook
-
-        parent = Process.objects.create(
-            machine=Machine.current(),
-            process_type=Process.TypeChoices.CLI,
-            status=Process.StatusChoices.RUNNING,
-        )
-        snap_dir = tmp_path / "parented-snapshot"
-        output_dir = snap_dir / "hashes"
-        output_dir.mkdir(parents=True)
-        (snap_dir / "source.txt").write_text("parented hook input", encoding="utf-8")
-        hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-
-        process = run_hook(
-            hook_path,
-            output_dir,
-            config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
-            timeout=30,
-            parent=parent,
-            url="https://example.com/parented-hook",
-        )
-
-        process.refresh_from_db()
-        assert process.exit_code == 0, process.stderr
-        assert process.parent_id == parent.id
-
 
 class TestRequiredBinaryConfigHandling:
     """Test that required_binaries keep configured XYZ_BINARY values intact."""
@@ -446,7 +417,7 @@ class TestHookExecution:
 
     @pytest.mark.django_db(transaction=True)
     def test_real_js_hook_runs_through_abxpkg_shebang(self, tmp_path, hermetic_lib_dir):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
         from archivebox.services.runner import run_install
 
         lib_dir = hermetic_lib_dir
@@ -456,7 +427,7 @@ class TestHookExecution:
         snap_dir = crawl_dir / "snapshot"
         hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
 
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             crawl_dir / "chrome",
             config={
@@ -551,7 +522,7 @@ class TestSnapshotHookOutput:
         [(True, "succeeded"), (False, "skipped")],
     )
     def test_hashes_hook_emits_real_archive_result(self, tmp_path, enabled, expected_status):
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / f"snapshot-{expected_status}"
         output_dir = snap_dir / "hashes"
@@ -559,7 +530,7 @@ class TestSnapshotHookOutput:
         (snap_dir / "source.txt").write_text("real hook protocol input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
 
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={
@@ -573,51 +544,21 @@ class TestSnapshotHookOutput:
         process.refresh_from_db()
 
         assert process.exit_code == 0, process.stderr
-        records = extract_records_from_process(process)
+        records = process.get_records()
         assert len(records) == 1
         assert records[0]["type"] == "ArchiveResult"
         assert records[0]["status"] == expected_status
-        assert records[0]["plugin"] == "hashes"
-        assert records[0]["hook_name"] == hook_path.name
-        assert records[0]["plugin_hook"] == str(hook_path)
+        assert process.cmd[0] == str(hook_path)
         if enabled:
             assert (output_dir / "hashes.json").is_file()
         else:
             assert records[0]["output_str"] == "HASHES_ENABLED=False"
 
 
-class TestPluginMetadata:
-    """Test that plugin metadata is added to JSONL records."""
-
-    @pytest.mark.django_db(transaction=True)
-    def test_python_hook_metadata_comes_from_executed_shipped_hook(self, tmp_path):
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
-
-        snap_dir = tmp_path / "snapshot-metadata"
-        output_dir = snap_dir / "hashes"
-        output_dir.mkdir(parents=True)
-        (snap_dir / "source.txt").write_text("metadata", encoding="utf-8")
-        script = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
-            script,
-            output_dir,
-            config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
-            timeout=30,
-            url="https://example.com/metadata",
-        )
-        process.refresh_from_db()
-
-        assert process.exit_code == 0, process.stderr
-        records = extract_records_from_process(process)
-        assert records[0]["plugin"] == "hashes"
-        assert records[0]["hook_name"] == script.name
-        assert records[0]["plugin_hook"] == str(script)
-
-
 @pytest.mark.django_db(transaction=True)
-def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_path, hermetic_lib_dir):
+def test_abx_dl_hook_execution_exports_singular_node_modules_dir_with_colon_node_path(tmp_path, hermetic_lib_dir):
     """Hook subprocesses must get a real NODE_MODULES_DIR even when NODE_PATH has multiple entries."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import run_install
 
     lib_dir = hermetic_lib_dir
@@ -632,7 +573,7 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
     crawl_dir = tmp_path / "crawl"
     output_dir = crawl_dir / "chrome"
     hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={
@@ -654,16 +595,16 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
 
 
 @pytest.mark.django_db(transaction=True)
-def test_run_hook_executes_python_hooks_through_abxpkg_shebang(tmp_path):
+def test_abx_dl_executes_python_hooks_through_abxpkg_shebang(tmp_path):
     """ArchiveBox treats Python hooks as opaque abxpkg-launched executables."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "snapshot"
     output_dir = snap_dir / "hashes"
     output_dir.mkdir(parents=True)
     (snap_dir / "source.txt").write_text("real runtime hook input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={

archivebox/tests/test_machine_service.py

@@ -13,6 +13,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 def _runtime_env(data_dir: Path, **extra: str) -> dict[str, str]:
     return {
         "ABXPKG_LIB_DIR": str(data_dir / "lib"),
+        "PLUGINS": "liteparse",
         "LITEPARSE_ENABLED": "True",
         "TIMEOUT": "180",
         "ABXPKG_INSTALL_TIMEOUT": "180",

archivebox/tests/test_migration_archiveresult_unique_plugin.py

@@ -1,53 +0,0 @@
-import pytest
-from django.db import connection
-from django.db.migrations.executor import MigrationExecutor
-
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-
-def test_migration_consolidates_plugin_rows_and_enforces_uniqueness():
-    try:
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0051_postgres_url_pattern_ops_index")])
-        old_apps = executor.loader.project_state([("core", "0051_postgres_url_pattern_ops_index")]).apps
-        Crawl = old_apps.get_model("crawls", "Crawl")
-        Snapshot = old_apps.get_model("core", "Snapshot")
-        ArchiveResult = old_apps.get_model("core", "ArchiveResult")
-        crawl = Crawl.objects.create(urls="https://example.com")
-        snapshot = Snapshot.objects.create(url="https://example.com/migration", crawl=crawl, output_size=10)
-        ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="responses",
-            hook_name="browser-upload",
-            status="succeeded",
-            output_files={"browser.png": {"size": 7}},
-            output_size=7,
-            output_mimetypes="image/png",
-        )
-        ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="responses",
-            hook_name="server-capture",
-            status="noresults",
-            output_files={"server.json": {"size": 3}},
-            output_size=3,
-            output_mimetypes="application/json",
-        )
-
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0052_unique_archiveresult_per_snapshot_plugin")])
-        new_apps = executor.loader.project_state([("core", "0052_unique_archiveresult_per_snapshot_plugin")]).apps
-        Snapshot = new_apps.get_model("core", "Snapshot")
-        ArchiveResult = new_apps.get_model("core", "ArchiveResult")
-        snapshot = Snapshot.objects.get(id=snapshot.id)
-        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="responses")
-
-        assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="responses").count() == 1
-        assert result.status == "succeeded"
-        assert set(result.output_files) == {"browser.png", "server.json"}
-        assert result.output_size == 10
-        assert snapshot.output_size == 10
-        assert set(result.output_mimetypes.split(",")) == {"application/json", "image/png"}
-    finally:
-        MigrationExecutor(connection).migrate([("core", "0052_unique_archiveresult_per_snapshot_plugin")])

archivebox/tests/test_migrations_07_to_09.py

@@ -403,7 +403,7 @@ def test_update_saves_migrated_snapshots_without_foreign_key_errors(archive_07):
     result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
     assert result.returncode == 0, f"Init failed: {result.stderr}"
 
-    result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=60)
+    result = run_archivebox_migration_cmd(work_dir, ["update", "--migrate-only"], timeout=60)
     output = result.stdout + result.stderr
     assert result.returncode == 0, f"Update failed after migration: {result.stderr}"
     assert "FOREIGN KEY constraint failed" not in output

archivebox/tests/test_migrations_08_to_09.py

@@ -898,7 +898,10 @@ def test_update_preserves_legacy_plugin_directory_without_output_files(migration
         (snapshot["id"],),
     ).fetchall()
     conn.close()
-    assert rows == [("", "{}", "")]
+    assert len(rows) == 2
+    assert all(output_str == "" for output_str, _output_files, _hook_name in rows)
+    assert len({hook_name for _output_str, _output_files, hook_name in rows}) == 2
+    assert sum(not hook_name for _output_str, _output_files, hook_name in rows) == 1
 
 
 def test_07_filesystem_hop_preserves_complete_output_tree(tmp_path):

archivebox/tests/test_process_runtime_paths.py

@@ -11,14 +11,14 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 class TestProcessRuntimePaths:
     def test_hook_processes_use_isolated_runtime_dir(self, tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "snapshot"
         output_dir = snap_dir / "hashes"
         output_dir.mkdir(parents=True)
         (snap_dir / "source.txt").write_text("real runtime path input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_recursive_crawl.py

@@ -493,10 +493,10 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
 ):
     """Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
 
-    from abx_dl.models import discover_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
     root_url = recursive_test_site["root_url"]
-    plugin_selection = ",".join(sorted(plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude")))
+    plugin_selection = ",".join(sorted(plugin for plugin in get_plugin_catalog() if not plugin.startswith("claude")))
     env = os.environ.copy()
     for preinstalled_path_key in (
         "CHROME_BINARY",

archivebox/tests/test_search.py

@@ -118,8 +118,8 @@ def configure_ripgrep_search_backend(lib_dir: Path) -> None:
     )
 
 
-def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
-    from archivebox.search.backends import search_backend_env
+def test_search_backend_command_env_serializes_config_without_mutating_process_env(tmp_path):
+    from archivebox.search.backends import search_backend_command_env
 
     old_env = os.environ.get("SEARCH_BACKEND_SONIC_HOST_NAME")
     os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] = "old-host"
@@ -136,15 +136,14 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
     )
 
     try:
-        with search_backend_env(config=config):
-            assert os.environ["SEARCH_BACKEND_ENGINE"] == "sonic"
-            assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
-            assert os.environ["SEARCH_BACKEND_SONIC_PORT"] == "1491"
-            assert os.environ["SEARCH_BACKEND_SONIC_PASSWORD"] == "SecretPassword"
-            assert os.environ["RIPGREP_BINARY"] == str(tmp_path / "env" / "bin" / "rg")
-            assert "IGNORED_NONE_VALUE" not in os.environ
-            assert "UNRELATED_BINARY_NAME" not in os.environ
-
+        env = search_backend_command_env(config=config)
+        assert env["SEARCH_BACKEND_ENGINE"] == "sonic"
+        assert env["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
+        assert env["SEARCH_BACKEND_SONIC_PORT"] == "1491"
+        assert env["SEARCH_BACKEND_SONIC_PASSWORD"] == "SecretPassword"
+        assert env["RIPGREP_BINARY"] == str(tmp_path / "env" / "bin" / "rg")
+        assert "IGNORED_NONE_VALUE" not in env
+        assert env["UNRELATED_BINARY_NAME"] == "rg"
         assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "old-host"
     finally:
         if old_env is None:

archivebox/tests/test_snapshot_service.py

@@ -40,7 +40,7 @@ def _snapshot_state(cwd: Path, url: str) -> dict[str, object]:
         }
 
 
-def test_snapshot_merge_consolidates_plugin_output_identity(admin_user):
+def test_snapshot_merge_consolidates_only_exact_hook_identity(admin_user):
     from archivebox.crawls.models import Crawl
 
     keeper_crawl = Crawl.objects.create(urls="https://example.com", created_by=admin_user)
@@ -64,15 +64,24 @@ def test_snapshot_merge_consolidates_plugin_output_identity(admin_user):
         output_files={"archivewebpage.wacz": {"size": 7}},
         output_size=7,
     )
+    ArchiveResult.objects.create(
+        snapshot=duplicate,
+        plugin="archivewebpage",
+        hook_name="on_Snapshot__16_archivewebpage_start",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="recording started",
+    )
+
     Snapshot._merge_snapshots([keeper, duplicate])
 
     assert not Snapshot.objects.filter(pk=duplicate.pk).exists()
     results = ArchiveResult.objects.filter(snapshot=keeper, plugin="archivewebpage")
-    assert results.count() == 1
-    stop_result = results.get()
+    assert results.count() == 2
+    stop_result = results.get(hook_name="on_Snapshot__65_archivewebpage_stop")
     assert stop_result.status == ArchiveResult.StatusChoices.SUCCEEDED
     assert stop_result.output_str == "archivewebpage.wacz"
     assert set(stop_result.output_files) == {"older.wacz", "archivewebpage.wacz"}
+    assert results.filter(hook_name="on_Snapshot__16_archivewebpage_start").exists()
 
 
 @pytest.mark.timeout(180)

archivebox/tests/test_ui_admin_archiveresult.py

@@ -142,8 +142,7 @@ class TestArchiveResultAdminListView:
         snapshot.refresh_from_db()
         assert snapshot.output_size == 0
 
-    def test_duplicate_plugin_result_is_rejected_without_touching_output(self, snapshot):
-        from django.db import IntegrityError, transaction
+    def test_deleting_sibling_hook_preserves_shared_plugin_output(self, snapshot):
         from archivebox.core.models import ArchiveResult
 
         output_dir = Path(snapshot.output_dir) / "responses"
@@ -157,13 +156,14 @@ class TestArchiveResultAdminListView:
             status=ArchiveResult.StatusChoices.SUCCEEDED,
             output_size=18,
         )
-        with pytest.raises(IntegrityError), transaction.atomic():
-            ArchiveResult.objects.create(
-                snapshot=snapshot,
-                plugin="responses",
-                hook_name="on_Snapshot__24_responses.daemon.bg.replayed",
-                status=ArchiveResult.StatusChoices.NORESULTS,
-            )
+        duplicate = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="responses",
+            hook_name="on_Snapshot__24_responses.daemon.bg.replayed",
+            status=ArchiveResult.StatusChoices.NORESULTS,
+        )
+
+        duplicate.delete()
 
         assert ArchiveResult.objects.filter(pk=primary.pk).exists()
         assert output_file.read_text() == "captured response\n"

archivebox/tests/test_ui_admin_links.py

@@ -15,13 +15,13 @@ from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 from archivebox.tests.conftest import install_real_binary
 
 
-pytestmark = pytest.mark.django_db
+pytestmark = pytest.mark.django_db(transaction=True)
 
 
 @pytest.fixture
 def real_hook_result(tmp_path):
     from archivebox.core.models import ArchiveResult
-    from archivebox.plugins.hooks import extract_records_from_process, run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snapshot = _create_snapshot()
     snap_dir = Path(snapshot.output_dir)
@@ -29,7 +29,7 @@ def real_hook_result(tmp_path):
     output_dir.mkdir(parents=True, exist_ok=True)
     (snap_dir / "source.txt").write_text("real admin link hook input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={
@@ -45,12 +45,12 @@ def real_hook_result(tmp_path):
     )
     process.refresh_from_db()
     assert process.exit_code == 0, process.stderr
-    record = extract_records_from_process(process)[0]
+    record = process.get_records()[0]
     hashes_file = output_dir / "hashes.json"
     result = ArchiveResult.objects.create(
         snapshot=snapshot,
-        plugin=record["plugin"],
-        hook_name=record["hook_name"],
+        plugin="hashes",
+        hook_name=hook_path.name,
         process=process,
         status=record["status"],
         output_str=record["output_str"],

archivebox/tests/test_ui_admin_machine.py

@@ -19,14 +19,14 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 @pytest.fixture
 def real_exited_hook_process(tmp_path):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "snapshot"
     output_dir = snap_dir / "hashes"
     output_dir.mkdir(parents=True)
     (snap_dir / "source.txt").write_text("real admin hook input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_ui_admin_snapshot.py

@@ -324,6 +324,35 @@ def test_snapshot_admin_preview_uses_extension_screenshot_when_standard_screensh
     assert "chrome_extension_screenshot/screenshot-2.png" not in preview["fallback_list"]
 
 
+def test_snapshot_admin_attributes_new_tags_to_authenticated_user(client, snapshot, admin_user):
+    from archivebox.core.models import Tag
+
+    client.force_login(admin_user)
+    response = client.post(
+        reverse("admin:core_snapshot_change", args=[snapshot.pk]),
+        {
+            "url": snapshot.url,
+            "title": snapshot.title or "",
+            "tags_editor": "admin-created-tag",
+            "permissions_config": "private",
+            "status": snapshot.status,
+            "retry_at": "",
+            "bookmarked_at_0": snapshot.bookmarked_at.date().isoformat(),
+            "bookmarked_at_1": snapshot.bookmarked_at.time().isoformat(),
+            "crawl": str(snapshot.crawl_id),
+            "config": '{"SAVE_ARCHIVE_DOT_ORG": "false"}',
+            "notes": "",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 302, response.context and response.context["adminform"].form.errors
+    tag = Tag.objects.get(name="admin-created-tag")
+    assert tag.created_by == admin_user
+    assert snapshot.tags.filter(pk=tag.pk).exists()
+
+
 class TestSnapshotProgressStats:
     """Tests for Snapshot.get_progress_stats() method."""
 
@@ -485,10 +514,46 @@ class TestSnapshotProgressStats:
 
         assert _count_media_files(result) == 2
         assert _list_media_files(result) == [
-            {"name": "audio.mp3", "path": "ytdlp/audio.mp3", "size": 222},
-            {"name": "video.mp4", "path": "ytdlp/video.mp4", "size": 111},
+            {
+                "name": "video.mp4",
+                "path": "ytdlp/video.mp4",
+                "size": 111,
+                "media_type": "video",
+                "is_video": True,
+                "is_audio": False,
+                "is_browser_playable": True,
+            },
+            {
+                "name": "audio.mp3",
+                "path": "ytdlp/audio.mp3",
+                "size": 222,
+                "media_type": "audio",
+                "is_video": False,
+                "is_audio": True,
+                "is_browser_playable": True,
+            },
         ]
 
+    def test_ytdlp_discover_outputs_prefers_browser_playable_video(self, snapshot):
+        from archivebox.core.models import ArchiveResult
+
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="ytdlp",
+            status="succeeded",
+            output_files={
+                "thumbnail.jpg": {"size": 20_000, "mimetype": "image/jpeg", "extension": "jpg"},
+                "large.mkv": {"size": 10_000, "mimetype": "video/x-matroska", "extension": "mkv"},
+                "small.mp4": {"size": 111, "mimetype": "video/mp4", "extension": "mp4"},
+            },
+            output_size=30_111,
+        )
+
+        outputs = snapshot.discover_outputs(include_filesystem_fallback=False)
+        ytdlp_output = next(output for output in outputs if output["name"] == "ytdlp")
+
+        assert ytdlp_output["path"] == "ytdlp/small.mp4"
+
     def test_discover_outputs_falls_back_to_hashes_index_without_filesystem_walk(
         self,
         snapshot,
@@ -928,10 +993,9 @@ class TestSnapshotOutputDeletion:
         from archivebox.core.models import ArchiveResult
 
         first = self._create_output(snapshot, size=11)
-        second = self._create_output(snapshot, plugin="singlefile", hook_name="on_Snapshot__50_singlefile.py", size=13)
+        second = self._create_output(snapshot, hook_name="on_Snapshot__51_screenshot_retry.py", size=13)
         kept = self._create_output(snapshot, plugin="pdf", hook_name="on_Snapshot__60_pdf.py", size=7)
         deleted_dir = Path(first.output_dir)
-        second_deleted_dir = Path(second.output_dir)
         kept_dir = Path(kept.output_dir)
         hashes_dir = Path(snapshot.output_dir) / "hashes"
         hashes_dir.mkdir(parents=True, exist_ok=True)
@@ -964,7 +1028,6 @@ class TestSnapshotOutputDeletion:
         assert not ArchiveResult.objects.filter(pk__in=[first.pk, second.pk]).exists()
         assert ArchiveResult.objects.filter(pk=kept.pk).exists()
         assert not deleted_dir.exists()
-        assert not second_deleted_dir.exists()
         assert kept_dir.exists()
         snapshot.refresh_from_db()
         assert snapshot.output_size == 7

archivebox/tests/test_ui_live_progress.py

@@ -21,14 +21,14 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 @pytest.fixture
 def real_unscoped_hook_process(tmp_path):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "snapshot"
     output_dir = snap_dir / "hashes"
     output_dir.mkdir(parents=True)
     (snap_dir / "source.txt").write_text("real live progress input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -55,7 +55,7 @@ def real_snapshot_hook_projection(snapshot, cached_abxpkg_lib_dir):
 
 @pytest.fixture
 def real_second_snapshot_hook_process(snapshot, tmp_path):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = Path(snapshot.output_dir)
     staticfile_dir = snap_dir / "staticfile"
@@ -64,7 +64,7 @@ def real_second_snapshot_hook_process(snapshot, tmp_path):
     output_dir.mkdir(parents=True, exist_ok=True)
     (staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -78,7 +78,7 @@ def real_second_snapshot_hook_process(snapshot, tmp_path):
 
 @pytest.fixture
 def real_crawl_setup_process(snapshot, hermetic_lib_dir):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import run_install
 
     hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
@@ -86,7 +86,7 @@ def real_crawl_setup_process(snapshot, hermetic_lib_dir):
     run_install(plugin_names=["chrome"])
     binary_env = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=config_path)
     output_dir = Path(snapshot.crawl.output_dir) / "chrome"
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={

archivebox/tests/test_urls.py

@@ -15,22 +15,26 @@ REPO_ROOT = Path(__file__).resolve().parents[2]
 
 
 @pytest.mark.django_db
-def test_archiveresult_relpath_uses_plugin_result_that_owns_output(admin_user):
+def test_archiveresult_relpath_uses_sibling_hook_that_owns_output(admin_user):
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.core.views import _resolve_archiveresult_relpath
     from archivebox.crawls.models import Crawl
 
     crawl = Crawl.objects.create(urls="https://example.com", created_by=admin_user)
     snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl)
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="screenshot",
+        hook_name="on_Snapshot__archivebox_browser_extension_upload",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_files={"browser.png": {"size": 7}},
+    )
     server_result = ArchiveResult.objects.create(
         snapshot=snapshot,
         plugin="screenshot",
         hook_name="on_Snapshot__50_screenshot",
         status=ArchiveResult.StatusChoices.SUCCEEDED,
-        output_files={
-            "browser.png": {"size": 7},
-            "screenshot.png": {"size": 6, "root_relative": True},
-        },
+        output_files={"screenshot.png": {"size": 6, "root_relative": True}},
     )
 
     resolved_path, result = _resolve_archiveresult_relpath(snapshot, "screenshot/screenshot.png")
@@ -134,6 +138,8 @@ def test_static_html_and_markdown_preview_images_rewrite_to_captured_responses(t
     assert "ETag" not in response.headers
     assert "max-age=60" in response.headers["Cache-Control"]
     assert b"archivebox-static-html-preview-style" in response.content
+    assert b"width: min(100%, 72rem)" in response.content
+    assert b"min-height: 100vh" in response.content
     assert b"img:not([width]):not([height])" in response.content
     assert b"a > img:not([width]):not([height])" in response.content
     assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content
@@ -163,6 +169,8 @@ def test_static_html_and_markdown_preview_images_rewrite_to_captured_responses(t
     assert "ETag" not in response.headers
     assert "max-age=60" in response.headers["Cache-Control"]
     assert b"archivebox-static-html-preview-style" in response.content
+    assert b"width: min(100%, 72rem)" in response.content
+    assert b"min-height: 100vh" in response.content
     assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content
 
 

bin/collect_ui_screenshots.sh

@@ -247,6 +247,11 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
     IFS='|' read -r name url expected_path source capture_mode <<<"$view"
     capture_index=$((capture_index + 1))
     slug="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//; s/-$//')"
+    expected_plugin=""
+    if [[ "$name" == "Snapshot View ("*")" && "$capture_mode" != "live-progress" && "$capture_mode" != "snapshot-collapsed" ]]; then
+        expected_plugin="${name#Snapshot View (}"
+        expected_plugin="${expected_plugin%)}"
+    fi
 
     echo "[*] $name: $url"
     if [[ "$capture_mode" == "snapshot-collapsed" ]]; then
@@ -382,6 +387,32 @@ PY
                 view_timing_report="$capture_dir/report.json"
                 timing_report_path="$view_timing_report"
             fi
+        elif [[ -n "$expected_plugin" && "$capture_mode" != wait-replay:* ]]; then
+            screenshot_path="$capture_dir/screenshot.png"
+            if [[ "$profile" == "desktop" ]]; then
+                mkdir -p \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile"
+                output_variants="$(printf \
+                    '[{"path":"%s","width":1600,"height":1000},{"path":"%s","width":1024,"height":1366},{"path":"%s","width":390,"height":844}]' \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop/screenshot.png" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet/screenshot.png" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile/screenshot.png")"
+                NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
+                    CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
+                    SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
+                    SCREENSHOT_WIDTH=1600 \
+                    SCREENSHOT_HEIGHT=1000 \
+                    SCREENSHOT_VARIANTS_JSON="$output_variants" \
+                    SCREENSHOT_COLLAPSE_FILTERS=1 \
+                    SCREENSHOT_EXPECT_PLUGIN="$expected_plugin" \
+                    node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
+                view_timing_report="$capture_dir/report.json"
+                timing_report_path="$view_timing_report"
+                uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
+                    "$capture_dir/report.json" "$expected_path"
+            fi
         elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then
             capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log"
             if ! env "${capture_env[@]}" uv run --no-cache --project "$REPO_DIR" abx-dl dl \
@@ -419,6 +450,7 @@ PY
                     SCREENSHOT_HEIGHT=1366 \
                     SCREENSHOT_VARIANTS_JSON="$responsive_variants" \
                     SCREENSHOT_COLLAPSE_FILTERS=1 \
+                    SCREENSHOT_EXPECT_PLUGIN="$expected_plugin" \
                     node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
                 view_timing_report="$capture_dir/report.json"
                 timing_report_path="$view_timing_report"

bin/take_screenshot.js

@@ -207,10 +207,14 @@ async function main() {
       await page.waitForFunction((pluginName) => {
         const selectedCard = document.querySelector('.thumb-card.selected-card[data-plugin-name]');
         const frame = document.querySelector('#main-frame');
+        const frameRect = frame?.getBoundingClientRect();
         return selectedCard?.dataset.pluginName?.toLowerCase() === pluginName
           && frame
           && frame.getAttribute('src')
-          && frame.getAttribute('src') !== 'about:blank';
+          && frame.getAttribute('src') !== 'about:blank'
+          && frameRect
+          && frameRect.width >= Math.min(window.innerWidth * 0.9, window.innerWidth - 8)
+          && frameRect.height >= Math.min(window.innerHeight * 0.35, 320);
       }, { timeout: 45000 }, expectedPlugin);
       await page.waitForFunction(() => {
         const frames = [document.querySelector('#main-frame'), document.querySelector('.thumb-card.selected-card iframe')]

docs/apidocs/archivebox/archivebox.services.runner.md

@@ -51,10 +51,6 @@
   - ```{autodoc2-docstring} archivebox.services.runner._is_nonfatal_setup_hook
     :summary:
     ```
-* - {py:obj}`_discover_archivebox_plugins <archivebox.services.runner._discover_archivebox_plugins>`
-  - ```{autodoc2-docstring} archivebox.services.runner._discover_archivebox_plugins
-    :summary:
-    ```
 * - {py:obj}`_runner_task_context <archivebox.services.runner._runner_task_context>`
   - ```{autodoc2-docstring} archivebox.services.runner._runner_task_context
     :summary:
@@ -63,10 +59,6 @@
   - ```{autodoc2-docstring} archivebox.services.runner._is_external_task_cancelled
     :summary:
     ```
-* - {py:obj}`_emit_machine_config <archivebox.services.runner._emit_machine_config>`
-  - ```{autodoc2-docstring} archivebox.services.runner._emit_machine_config
-    :summary:
-    ```
 * - {py:obj}`_run_event_now <archivebox.services.runner._run_event_now>`
   - ```{autodoc2-docstring} archivebox.services.runner._run_event_now
     :summary:
@@ -251,13 +243,6 @@
 ```
 ````
 
-````{py:function} _discover_archivebox_plugins() -> dict[str, abx_dl.models.Plugin]
-:canonical: archivebox.services.runner._discover_archivebox_plugins
-
-```{autodoc2-docstring} archivebox.services.runner._discover_archivebox_plugins
-```
-````
-
 ````{py:function} _runner_task_context() -> contextvars.Context
 :canonical: archivebox.services.runner._runner_task_context
 
@@ -272,14 +257,6 @@
 ```
 ````
 
-````{py:function} _emit_machine_config(bus, *, config: dict[str, typing.Any], derived_config: dict[str, typing.Any], parent_event=None) -> None
-:canonical: archivebox.services.runner._emit_machine_config
-:async:
-
-```{autodoc2-docstring} archivebox.services.runner._emit_machine_config
-```
-````
-
 ````{py:function} _run_event_now(event, timeout: float | None = None)
 :canonical: archivebox.services.runner._run_event_now
 :async:
@@ -342,14 +319,6 @@
 
 ````
 
-````{py:method} runtime_plugins() -> dict[str, abx_dl.models.Plugin]
-:canonical: archivebox.services.runner.CrawlRunner.runtime_plugins
-
-```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.runtime_plugins
-```
-
-````
-
 ````{py:property} allow_maintenance_on_inactive_crawl
 :canonical: archivebox.services.runner.CrawlRunner.allow_maintenance_on_inactive_crawl
 :type: bool

docs/codeblocks.toml

@@ -74,8 +74,8 @@ version = 2
 # docs/ArchiveBox-Architecture-Diagrams.md
 "7442dd337fd0a1dd-1" = "illustration"
 "9d6f8ac3043ff691-1" = "illustration"
-"8c8802244e05cfb7-1" = "illustration"
-"d0eb428affe5bb44-1" = "illustration"
+"9529bb7bc396b6b0-1" = "illustration"
+"66018350af060c17-1" = "illustration"
 "7d1d068e1e754a75-1" = "illustration"
 
 # docs/Changelog.md

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc372",
+  "version": "0.9.35rc404",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc372"
+version = "0.9.35rc404"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (runtime binaries resolved through abxpkg)
     ### Binary/Package Management
     "abxbus==2.5.56",                 # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abxpkg==1.12.110",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins==1.12.199",           # direct imports only; version constrained by abx-dl
-    "abx-dl==1.12.227",       # shared ArchiveBox downloader package
+    "abxpkg==1.12.117",                # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abx-plugins==1.12.221",           # direct imports only; version constrained by abx-dl
+    "abx-dl==1.12.256",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]
@@ -342,7 +342,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.35rc372"
+current_version = "v0.9.35rc404"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-08-27T20:10:49.457673934Z"
+exclude-newer = "2026-08-28T20:27:25.177666929Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-01T20:10:48.457686597Z", span = "PT1S" }
+abxbus = { timestamp = "2026-09-02T20:27:24.177684316Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.12.227"
+version = "1.12.256"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/d3/de/34953ddf681c30e9daca9fd1c25936c3f52955098e709d3270673792911a/abx_dl-1.12.227.tar.gz", hash = "sha256:211080fe318dd872dd9754d296a5569e03c478ec415ac386fb92252e963d8479", size = 88731 }
+sdist = { url = "https://files.pythonhosted.org/packages/7a/80/c7c2555d3043b5b1c1e902fc1054941e8f866e5966146da84677637b54b1/abx_dl-1.12.256.tar.gz", hash = "sha256:f94da3ccbe6153aac46a9976d3de75fe5b6d2b576487c361e76fcbc2c967146b", size = 86752 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/6d/fb/d8be5a4fe9abe40fecb91219041bc6b10746127c9887fec032f446cc751f/abx_dl-1.12.227-py3-none-any.whl", hash = "sha256:4741554a429614733694b0d34c8632e3909f719efe46b04da6145d586e043b26", size = 93102 },
+    { url = "https://files.pythonhosted.org/packages/8d/1e/0d0cd296587ec1390ddc166fec3bfbeca69845e9b06345cc1dd0ae13161a/abx_dl-1.12.256-py3-none-any.whl", hash = "sha256:091ac73f0ccf9eab18397897e1d46760df076106b3e2be16741e1145e2e66f49", size = 91257 },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.12.199"
+version = "1.12.221"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/71/62/67551f1fbd77a3ed60b844725be0ef9c1525d3e2acc1cf0ff704373d2520/abx_plugins-1.12.199.tar.gz", hash = "sha256:1119a7fa545bf78a36a5081d1537bbc765faae560101a61e2d2349fd8a44e689", size = 259750, upload-time = "2026-09-01T19:58:59.103Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/99/06/5c806420d1267d9ba56cccd9f2ca2617e7aae0fe5fbef9678d08b7c9e0d6/abx_plugins-1.12.221.tar.gz", hash = "sha256:6d42a4eaf7cd8515977a23de9172ae05366243a4c4fb9b4d7d8d1acc6849fd3b", size = 258487, upload-time = "2026-09-02T20:19:21.765Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/fc/672c553894f96ee934b9ba502e040cf065ef893a294326239033e76fa533/abx_plugins-1.12.199-py3-none-any.whl", hash = "sha256:35a5db4d953403f6e410f1da97f9f9177251e10afdd9060450f7737ec88d9bbc", size = 413582, upload-time = "2026-09-01T19:58:57.502Z" },
+    { url = "https://files.pythonhosted.org/packages/05/fb/cf9c05a08bb006963b246ccce75dd3f6052954b7f110bb1f89ef02ff56d8/abx_plugins-1.12.221-py3-none-any.whl", hash = "sha256:43e2ce175c6ff4cc87d049b7b427ae22649e44bf2daca3de6e34ab6172acb337", size = 412364, upload-time = "2026-09-02T20:19:20.1Z" },
 ]
 
 [[package]]
@@ -75,7 +75,7 @@ wheels = [
 
 [[package]]
 name = "abxpkg"
-version = "1.12.110"
+version = "1.12.117"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -83,9 +83,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/dc/25/152a0e2dd37494cab5116629b4a3f26232dda6641e7b79f8eb7d1e9bfd75/abxpkg-1.12.110.tar.gz", hash = "sha256:94777e6ea76944b6577c46aaf4d845cdc68cfbdfe4c58ca24cd02071e367a660", size = 239950, upload-time = "2026-08-31T01:25:16.393Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/35/bbfaccae109328c716cea4e0cdbf2d2dc5975f448203bb3f4daa605591a6/abxpkg-1.12.117.tar.gz", hash = "sha256:4d88cc293f343eee2be7ecaab1203dfbc20571a4a39b2afa2f866c9ef8032dec", size = 239387, upload-time = "2026-09-02T19:57:26.142Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/05/95/49a113ceccf1d1ac33a90a0f8735abd43919f6cf2543917542acbdd61ea3/abxpkg-1.12.110-py3-none-any.whl", hash = "sha256:01e3e95c0af1a8fe775dee07eb16fa540a4e7007c3961a52e11d57bc2eae2430", size = 254023, upload-time = "2026-08-31T01:25:14.833Z" },
+    { url = "https://files.pythonhosted.org/packages/73/46/372be1f8fa915eb254ad865b97fb41073acdb5573dbce3e23a6083417d87/abxpkg-1.12.117-py3-none-any.whl", hash = "sha256:76ec8afaac7b2d7aece7bdda2bbf606e6373db51eb4dcb906008052d09e59577", size = 253749, upload-time = "2026-09-02T19:57:27.437Z" },
 ]
 
 [[package]]
@@ -120,7 +120,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.35rc372"
+version = "0.9.35rc404"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -219,10 +219,10 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.12.227" },
-    { name = "abx-plugins", specifier = "==1.12.199" },
+    { name = "abx-dl", specifier = "==1.12.256" },
+    { name = "abx-plugins", specifier = "==1.12.221" },
     { name = "abxbus", specifier = "==2.5.56" },
-    { name = "abxpkg", specifier = "==1.12.110" },
+    { name = "abxpkg", specifier = "==1.12.117" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
     { name = "atomicwrites", specifier = "==1.4.1" },
     { name = "base32-crockford", specifier = ">=0.3.0" },