Bump release version to 0.9.35rc405

Supersedes #1818.

This keeps yt-dlp-specific player rendering out of ArchiveBox core. Core
now:

- exposes media type metadata to plugin card templates: `is_video`,
`is_audio`, `is_browser_playable`
- sorts yt-dlp media with browser-playable video/audio first
- prefers browser-playable yt-dlp outputs before thumbnails and
non-browser containers when choosing the primary card output
- avoids pinning resumed snapshots to stale selected hook maps, fixing a
pause/resume runner regression surfaced by CI
- makes `update --index-only` discover the configured search backend
even when `PLUGINS` is restricted for test/runtime isolation
- stabilizes CI-only recursive/takeover/public-UI tests that were
failing due external/heavy plugin coverage, hard-kill recovery races,
and transient full-server `/add/` startup responses

The actual click-to-load player UI lives in the companion plugin PR:
ArchiveBox/abx-plugins#34.

Verification:
```text
pytest archivebox/tests/test_ui_admin_snapshot.py -q
48 passed

pytest archivebox/tests/test_cli_update_reindex_snapshots.py -q
9 passed

pytest -xvs archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py --basetemp=tests/out --ignore=archivebox/pkgs
7 passed

pytest -xvs archivebox/tests/test_takeover_util.py --basetemp=tests/out --ignore=archivebox/pkgs
19 passed

pytest -xvs archivebox/tests/test_takeover_util.py::test_live_add_update_jobs_survive_server_and_cli_owner_exits --basetemp=tests/out --ignore=archivebox/pkgs
1 passed

pytest -xvs archivebox/tests/test_ui_public_snapshot.py --basetemp=tests/out --ignore=archivebox/pkgs
6 passed

pytest -q archivebox/tests/test_recursive_crawl.py::test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel --basetemp=tests/out --ignore=archivebox/pkgs
1 passed
```

.github/workflows/docs.yml

@@ -99,9 +99,10 @@ jobs:
           if [[ '${{ matrix.environment }}' != macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docs_binaries")
           fi
-          if [[ '${{ matrix.environment }}' == ubuntu ]]; then
+          if [[ '${{ matrix.name }}' == ubuntu-AGENTS || '${{ matrix.name }}' == ubuntu-skills-archivebox-SKILL || '${{ matrix.name }}' == ubuntu-docs-Configuration ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-linux-build.json:required_binaries")
-          elif [[ '${{ matrix.environment }}' == docker ]]; then
+          fi
+          if [[ '${{ matrix.environment }}' == docker ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-tooling.json:docker_binaries")
           elif [[ '${{ matrix.environment }}' == macos ]]; then
             dependency_sources+=(--deps-from="$GITHUB_WORKSPACE/.github/configs/ci-macos-brew.json:required_binaries")

AGENTS.md

@@ -15,6 +15,13 @@ ArchiveBox is the full self-hosted web archiving app. Keep this repo on the `dev
 - Trace root causes from observed behavior. Do not paper over failures with retries, wider timeouts, broad fallbacks, or looser assertions.
 - Read `README.md` for the full setup, CLI, Docker, API, and release surface.
 
+## Concurrency Contract
+
+- A collection has one orchestrator at a time. Local PID/process checks may warn about obvious same-machine duplicates, but must not claim to enforce ownership across machines or shared filesystems.
+- SQLite remains supported with concurrent short writes from CLI and server processes. Keep the SQLite database on a local filesystem, never NFS/SMB, and use short autocommit/CAS updates instead of long transactions or database locks.
+- Network calls, hook execution, filesystem migrations, and other long work belong in the orchestrator. Never hold a database transaction or lock across that work.
+- PostgreSQL and shared data directories are the path to future multi-machine scheduling. Coordinate that work through database-backed per-crawl/per-snapshot claims; do not introduce file leases or timer-based orchestrator election.
+
 ## Development Setup
 
 ```bash

Dockerfile

@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.225 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.256 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.225
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.256
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base

archivebox/api/v1_core.py

@@ -11,7 +11,7 @@ from uuid import UUID
 from typing import Union, Any, Annotated
 from datetime import datetime, time
 
-from django.db import transaction
+from django.db import IntegrityError
 from django.db.models import Model, Q
 from django.http import HttpRequest, HttpResponse
 from django.http.multipartparser import MultiPartParser, MultiPartParserError
@@ -342,21 +342,10 @@ def _parse_archiveresult_upload_int(value: str, field_name: str, *, default: int
 
 
 def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]]) -> tuple[int, str]:
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-    for metadata in output_files.values():
-        if not isinstance(metadata, dict):
-            continue
-        try:
-            size = max(int(metadata.get("size") or 0), 0)
-        except (TypeError, ValueError):
-            size = 0
-        mime_type = str(metadata.get("mimetype") or "").strip()
-        total_size += size
-        if mime_type and size:
-            mime_sizes[mime_type] += size
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return total_size, output_mimetypes
+    from abx_dl.output_files import OutputManifest
+
+    manifest = OutputManifest.from_value(output_files)
+    return manifest.total_size, ",".join(manifest.mimetypes)
 
 
 def _get_snapshot_by_ref(snapshot_id: str):
@@ -528,52 +517,53 @@ def create_archiveresult(
     normalized_status = ArchiveResult.normalize_status(status)
     parsed_output_json = _parse_archiveresult_output_json(output_json)
     hook = hook_name or ARCHIVERESULT_UPLOAD_HOOK_NAME
-    existing_result = ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin_name).first()
-    existing_output_files = dict(existing_result.output_files or {}) if existing_result else {}
-    output_files = _write_archiveresult_files(
+    result_lookup = {
+        "snapshot": snapshot,
+        "plugin": plugin_name,
+        "hook_name": hook,
+    }
+    uploaded_output_files = _write_archiveresult_files(
         request,
         snapshot,
         plugin_name,
-        existing_output_files=existing_output_files,
         allow_empty=True,
     )
-    now = timezone.now()
-
-    with transaction.atomic():
-        Snapshot.objects.select_for_update().get(pk=snapshot.pk)
-        existing_result = ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin_name).first()
-        if existing_result:
-            output_files = {
-                **dict(existing_result.output_files or {}),
-                **output_files,
-            }
-            result = existing_result
-        else:
-            existing_result = None
-            result = ArchiveResult(
-                snapshot=snapshot,
-                plugin=plugin_name,
-                hook_name=hook,
-            )
-
-        if (
-            existing_result
-            and normalized_status == ArchiveResult.StatusChoices.STARTED
-            and existing_result.status != ArchiveResult.StatusChoices.STARTED
-        ):
-            normalized_status = existing_result.status
+    result = ArchiveResult.objects.filter(**result_lookup).first()
+    for _attempt in range(3):
+        output_files = {
+            **(result.output_file_map() if result else {}),
+            **uploaded_output_files,
+        }
         output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
         output_file_paths = list(output_files.keys())
-        result.status = normalized_status
-        result.hook_name = hook
-        result.output_str = output_str or (output_file_paths[0] if output_file_paths else "")
-        result.output_json = parsed_output_json
-        result.output_files = output_files
-        result.output_size = output_size
-        result.output_mimetypes = output_mimetypes
-        result.start_ts = result.start_ts or now
-        result.end_ts = now
-        result.save()
+        result_status = normalized_status
+        if result and result_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+            result_status = result.status
+        now = timezone.now()
+        values = {
+            "status": result_status,
+            "output_str": output_str or (output_file_paths[0] if output_file_paths else ""),
+            "output_json": parsed_output_json,
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "start_ts": result.start_ts or now if result else now,
+            "end_ts": now,
+        }
+        if result:
+            if result.safe_update(values):
+                break
+            continue
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_name,
+            hook,
+            defaults=values,
+        )
+        if created:
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
 
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(snapshot)
@@ -587,44 +577,46 @@ def patch_archiveresult(
 ):
     """Append or replace files on an existing ArchiveResult."""
     result = ArchiveResult.objects.select_related("snapshot__crawl__created_by").get(_uuid_ref_query("id", archiveresult_id))
-    output_files = _write_archiveresult_files(
+    uploaded_output_files = _write_archiveresult_files(
         request,
         result.snapshot,
         result.plugin,
-        existing_output_files=result.output_file_map(),
     )
-    latest_result = ArchiveResult.objects.only("output_files", "status").get(pk=result.pk)
-    output_files = {
-        **latest_result.output_file_map(),
-        **output_files,
-    }
-    output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
-
-    update_fields = ["output_files", "output_size", "output_mimetypes", "end_ts", "modified_at"]
-    result.output_files = output_files
-    result.output_size = output_size
-    result.output_mimetypes = output_mimetypes
-    result.end_ts = timezone.now()
     output_str = _get_archiveresult_upload_form_value(request, "output_str")
     status = _get_archiveresult_upload_form_value(request, "status")
     output_json = _get_archiveresult_upload_form_value(request, "output_json")
-    if output_str:
-        result.output_str = output_str
-        update_fields.append("output_str")
-    if status:
-        normalized_status = ArchiveResult.normalize_status(status)
-        if normalized_status == ArchiveResult.StatusChoices.STARTED and latest_result.status != ArchiveResult.StatusChoices.STARTED:
-            normalized_status = latest_result.status
-        result.status = normalized_status
-        update_fields.append("status")
-    elif latest_result.status == ArchiveResult.StatusChoices.QUEUED and ArchiveResult.output_files_upload_complete(output_files):
-        result.status = ArchiveResult.StatusChoices.SUCCEEDED
-        update_fields.append("status")
-    if output_json:
-        result.output_json = _parse_archiveresult_output_json(output_json)
-        update_fields.append("output_json")
+    parsed_output_json = _parse_archiveresult_output_json(output_json) if output_json else None
+
+    if not ArchiveResult.output_files_upload_complete(uploaded_output_files):
+        result.output_files = {**result.output_file_map(), **uploaded_output_files}
+        result.output_size, result.output_mimetypes = _summarize_archiveresult_output_files(result.output_files)
+        return result
+
+    for _attempt in range(3):
+        output_files = {**result.output_file_map(), **uploaded_output_files}
+        output_size, output_mimetypes = _summarize_archiveresult_output_files(output_files)
+        values: dict[str, Any] = {
+            "output_files": output_files,
+            "output_size": output_size,
+            "output_mimetypes": output_mimetypes,
+            "end_ts": timezone.now(),
+        }
+        if output_str:
+            values["output_str"] = output_str
+        if status:
+            normalized_status = ArchiveResult.normalize_status(status)
+            if normalized_status == ArchiveResult.StatusChoices.STARTED and result.status != ArchiveResult.StatusChoices.STARTED:
+                normalized_status = result.status
+            values["status"] = normalized_status
+        elif result.status == ArchiveResult.StatusChoices.QUEUED:
+            values["status"] = ArchiveResult.StatusChoices.SUCCEEDED
+        if output_json:
+            values["output_json"] = parsed_output_json
+        if result.safe_update(values):
+            break
+    else:
+        raise HttpError(409, "ArchiveResult changed while upload metadata was being updated")
 
-    result.save(update_fields=update_fields)
     if result.status != ArchiveResult.StatusChoices.STARTED:
         _queue_archiveresult_snapshot_maintenance(result.snapshot)
 
@@ -941,52 +933,47 @@ def create_snapshot(request: HttpRequest, data: SnapshotCreateSchema):
             created_by=request.user if isinstance(request.user, User) else None,
         )
 
-    # Browser clients sync metadata and upload capture artifacts immediately
-    # after queueing a URL. The runner holds the crawl lifecycle lock for the
-    # whole crawl, so idempotently posting an already-created snapshot must not
-    # wait behind that potentially long-running archive job.
-    existing_snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
-    if existing_snapshot is not None and (status is None or existing_snapshot.status == status):
-        if data.title is not None and existing_snapshot.title != data.title:
-            Snapshot.objects.filter(pk=existing_snapshot.pk).update(title=data.title, modified_at=timezone.now())
-            existing_snapshot.title = data.title
-        if tags:
-            existing_snapshot.save_tags(tags)
-        setattr(request, "with_archiveresults", False)
-        return existing_snapshot
+    # Browser uploads must not wait behind the runner's crawl-wide lifecycle
+    # lock. The unique insert recovery and CAS update below are the request-side
+    # coordination boundary for this idempotent metadata sync.
+    snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+    if snapshot is None:
+        try:
+            snapshot = Snapshot.objects.create(
+                url=data.url,
+                crawl=crawl,
+                depth=data.depth,
+                title=data.title,
+                timestamp=str(timezone.now().timestamp()),
+                status=status or Snapshot.StatusChoices.QUEUED,
+                retry_at=timezone.now(),
+            )
+        except IntegrityError:
+            snapshot = Snapshot.objects.filter(url=data.url, crawl=crawl).first()
+            if snapshot is None:
+                raise
 
-    with crawl_lifecycle_lock(str(crawl.id)):
-        snapshot_defaults = {
-            "depth": data.depth,
-            "title": data.title,
-            "timestamp": str(timezone.now().timestamp()),
-            "status": status or Snapshot.StatusChoices.QUEUED,
-            "retry_at": timezone.now(),
-        }
-        snapshot, _ = Snapshot.objects.get_or_create(
-            url=data.url,
-            crawl=crawl,
-            defaults=snapshot_defaults,
+    for _attempt in range(3):
+        updates: dict[str, Any] = {}
+        if data.title is not None and snapshot.title != data.title:
+            updates["title"] = data.title
+        if status is not None and snapshot.status != status:
+            updates["status"] = status
+        if not updates or snapshot.safe_update(updates, extra_filter={"modified_at": snapshot.modified_at}):
+            break
+    else:
+        raise HttpError(409, "Snapshot changed while metadata was being updated")
+
+    if tags:
+        snapshot.save_tags(
+            tags,
+            created_by=request.user if isinstance(request.user, User) else None,
         )
 
-        update_fields: list[str] = []
-        if data.title is not None and snapshot.title != data.title:
-            snapshot.title = data.title
-            update_fields.append("title")
-        if status is not None and snapshot.status != status:
-            snapshot.status = status
-            update_fields.append("status")
-        if update_fields:
-            update_fields.append("modified_at")
-            snapshot.save(update_fields=update_fields)
-
-        if tags:
-            snapshot.save_tags(tags)
-
-        try:
-            snapshot.ensure_crawl_symlink()
-        except Exception:
-            pass
+    try:
+        snapshot.ensure_crawl_symlink()
+    except Exception:
+        pass
 
     setattr(request, "with_archiveresults", False)
     return snapshot
@@ -1034,7 +1021,10 @@ def patch_snapshot(request: HttpRequest, snapshot_id: str, data: SnapshotUpdateS
             update_fields.append("retry_at")
 
         if tags is not None:
-            snapshot.save_tags(normalize_tag_list(tags))
+            snapshot.save_tags(
+                normalize_tag_list(tags),
+                created_by=request.user if isinstance(request.user, User) else None,
+            )
 
         if payload.get("status") == Snapshot.StatusChoices.SEALED:
             snapshot.cancel()
@@ -1421,8 +1411,7 @@ def tags_add_to_snapshot(request: HttpRequest, data: TagSnapshotRequestSchema):
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    # Add the tag to the snapshot
-    snapshot.tags.add(tag.pk)
+    snapshot.add_tag_ids([tag.pk])
 
     return {
         "success": True,
@@ -1450,8 +1439,7 @@ def tags_remove_from_snapshot(request: HttpRequest, data: TagSnapshotRequestSche
     else:
         raise HttpError(400, "Either tag_name or tag_id is required")
 
-    # Remove the tag from the snapshot
-    snapshot.tags.remove(tag.pk)
+    snapshot.remove_tag_ids([tag.pk])
 
     return {
         "success": True,

archivebox/cli/archivebox_add.py

@@ -213,7 +213,7 @@ def add(
     print(f"[green]\\[+] Created Crawl {crawl.id} with max_depth={depth}[/green]")
     print(f"    [dim]First URL: {first_url}[/dim]")
 
-    # 3. The CrawlMachine will create Snapshots from all URLs when started
+    # 3. The runner will create Snapshots from all URLs after claiming the Crawl
     #    Parser extractors run on snapshots and discover more URLs
     #    Discovered URLs become child Snapshots (depth+1)
 
@@ -421,13 +421,13 @@ def main(**kwargs):
         if only_new is not None:
             kwargs["config"] = {"ONLY_NEW": bool(only_new)}
         if extract:
-            from abx_dl.models import discover_plugins, plugins_matching_output
+            from archivebox.plugins.discovery import get_plugin_catalog
 
-            all_plugins = discover_plugins()
+            catalog = get_plugin_catalog()
             tokens = [token.strip() for token in extract.split(",") if token.strip()]
-            plugin_names = {name.lower(): name for name in all_plugins}
+            plugin_names = {name.lower(): name for name in catalog}
             selected = [plugin_names[token.lower()] for token in tokens if token.lower() in plugin_names]
-            selected += plugins_matching_output(all_plugins, tokens)
+            selected += catalog.matching_output(tokens)
             if not selected:
                 raise click.UsageError(f"No plugins found matching extract types: {extract}")
             existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]

archivebox/cli/archivebox_archiveresult.py

@@ -67,7 +67,7 @@ def create_archiveresults(
     Reads Snapshot records from stdin and emits ArchiveResult request JSONL.
     Pass-through: Non-Snapshot/ArchiveResult records are output unchanged.
     If --plugin is specified, only emits requests for that plugin.
-    Otherwise, emits one request for each enabled Snapshot plugin.
+    Otherwise, emits requests for all enabled snapshot hooks.
 
     Exit codes:
         0: Success
@@ -144,17 +144,18 @@ def create_archiveresults(
 
     created_count = 0
     for snapshot in snapshots:
-        if plugin:
+        config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
+        hooks = [
+            hook
+            for hook in discover_hooks("Snapshot", filter_disabled=not plugin, config=config)
+            if not plugin or hook.parent.name == plugin
+        ]
+        for hook_path in hooks:
+            hook_name = hook_path.stem
+            plugin_name = hook_path.parent.name
             if not is_tty:
-                write_record(build_archiveresult_request(snapshot.id, plugin, status=status))
+                write_record(build_archiveresult_request(snapshot.id, plugin_name, hook_name=hook_name, status=status))
             created_count += 1
-        else:
-            config = get_config(crawl=snapshot.crawl, snapshot=snapshot)
-            hooks = discover_hooks("Snapshot", filter_disabled=True, config=config)
-            for plugin_name in dict.fromkeys(hook_path.parent.name for hook_path in hooks):
-                if not is_tty:
-                    write_record(build_archiveresult_request(snapshot.id, plugin_name, status=status))
-                created_count += 1
 
     rprint(f"[green]Created {created_count} archive result request records[/green]", file=sys.stderr)
     return 0
@@ -254,11 +255,9 @@ def update_archiveresults(
 
             # Apply updates from CLI flags
             if status:
-                if status == ArchiveResult.StatusChoices.QUEUED:
-                    result.reset_for_retry()
-                else:
-                    result.status = status
-                    result.save(update_fields=["status", "modified_at"])
+                result.status = status
+
+            result.save()
             updated_count += 1
 
             if not is_tty:

archivebox/cli/archivebox_extract.py

@@ -70,7 +70,12 @@ def process_archiveresult_by_id(archiveresult_id: str) -> int:
             )
             return 1
 
-        run_crawl(str(snapshot.crawl_id), snapshot_ids=[str(snapshot.id)], selected_plugins=[archiveresult.plugin])
+        run_crawl(
+            str(snapshot.crawl_id),
+            snapshot_ids=[str(snapshot.id)],
+            selected_plugins=[archiveresult.plugin],
+            selected_plugins_are_explicit=False,
+        )
         archiveresult.refresh_from_db()
 
         if archiveresult.status == ArchiveResult.StatusChoices.SUCCEEDED:
@@ -122,6 +127,7 @@ def run_plugins(
     from archivebox.core.models import Snapshot
     from archivebox.core.models import ArchiveResult
     from archivebox.services.runner import run_crawl
+    from archivebox.plugins.discovery import get_plugin_catalog
 
     is_tty = sys.stdout.isatty()
 
@@ -142,6 +148,8 @@ def run_plugins(
     # Gather snapshot IDs and optional plugin constraints to process
     snapshot_ids = set()
     requested_plugins_by_snapshot: dict[str, set[str]] = defaultdict(set)
+    requested_hooks_by_snapshot: dict[str, set[tuple[str, str]]] = defaultdict(set)
+    plugin_level_requests_by_snapshot: dict[str, set[str]] = defaultdict(set)
     for record in records:
         record_type = record.get("type")
 
@@ -163,7 +171,14 @@ def run_plugins(
                 snapshot_ids.add(str(snapshot_id))
                 plugin_name = record.get("plugin")
                 if plugin_name and not plugins_list:
-                    requested_plugins_by_snapshot[str(snapshot_id)].add(str(plugin_name))
+                    snapshot_key = str(snapshot_id)
+                    plugin_key = str(plugin_name)
+                    requested_plugins_by_snapshot[snapshot_key].add(plugin_key)
+                    hook_name = str(record.get("hook_name") or "")
+                    if hook_name:
+                        requested_hooks_by_snapshot[snapshot_key].add((plugin_key, hook_name))
+                    else:
+                        plugin_level_requests_by_snapshot[snapshot_key].add(plugin_key)
 
         elif "id" in record:
             # Assume it's a snapshot ID
@@ -192,21 +207,38 @@ def run_plugins(
             if snapshot_id in existing_snapshot_ids
             for plugin_name in plugin_names
         )
-    queued_rows: set[tuple[str, str]] = set()
-    if preserve_queued and requested_pairs:
-        queued_rows = {
-            (str(snapshot_id), plugin_name)
-            for snapshot_id, plugin_name in ArchiveResult.objects.filter(
-                snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name in requested_pairs},
-                status=ArchiveResult.StatusChoices.QUEUED,
-            ).values_list("snapshot_id", "plugin")
+    plugins_by_name = get_plugin_catalog()
+    requested_rows: set[tuple[str, str, str]] = set()
+    for snapshot_id, plugin_name in requested_pairs:
+        exact_hook_names = {
+            hook_name
+            for requested_plugin, hook_name in requested_hooks_by_snapshot.get(snapshot_id, set())
+            if requested_plugin == plugin_name
         }
-    rows_to_queue = requested_pairs - queued_rows
+        if exact_hook_names and plugin_name not in plugin_level_requests_by_snapshot.get(snapshot_id, set()):
+            requested_rows.update((snapshot_id, plugin_name, hook_name) for hook_name in exact_hook_names)
+            continue
+        plugin = plugins_by_name.get(plugin_name)
+        hooks = plugin.filter_hooks("Snapshot") if plugin is not None else []
+        if hooks:
+            requested_rows.update((snapshot_id, plugin_name, hook.name) for hook in hooks)
+        else:
+            requested_rows.add((snapshot_id, plugin_name, ""))
+
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
 
     reset_fields = {
         "status": ArchiveResult.StatusChoices.QUEUED,
-        "hook_name": "",
         "output_str": "",
         "output_json": None,
         "output_files": {},
@@ -217,28 +249,28 @@ def run_plugins(
         "modified_at": timezone.now(),
     }
     if rows_to_queue and plugins_list:
-        rows_to_reset_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name in rows_to_queue:
-            rows_to_reset_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in rows_to_reset_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
                 **reset_fields,
             )
     elif rows_to_queue and requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_name in rows_to_queue:
-            snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
                 **reset_fields,
             )
     existing_rows = (
         {
-            (str(snapshot_id), plugin_name)
-            for snapshot_id, plugin_name in ArchiveResult.objects.filter(
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
                 snapshot_id__in=existing_snapshot_ids,
-                plugin__in={plugin_name for _snapshot_id, plugin_name in rows_to_queue},
-            ).values_list("snapshot_id", "plugin")
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
         }
         if rows_to_queue
         else set()
@@ -250,10 +282,10 @@ def run_plugins(
                 ArchiveResult(
                     snapshot_id=snapshot_id,
                     plugin=plugin_name,
-                    hook_name="",
+                    hook_name=hook_name,
                     status=ArchiveResult.StatusChoices.QUEUED,
                 )
-                for snapshot_id, plugin_name in sorted(missing_rows)
+                for snapshot_id, plugin_name, hook_name in sorted(missing_rows)
             ],
             batch_size=500,
         )
@@ -261,13 +293,12 @@ def run_plugins(
     processed_count = len(existing_snapshot_ids)
     queue_at = timezone.now()
     if existing_snapshot_ids:
-        if requested_pairs:
-            # Search indexing on a sealed Snapshot is the only targeted plugin
-            # allowed to bypass the normal lifecycle. Every other requested
-            # plugin requeues its Snapshot through the unified state machine.
-            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name in rows_to_queue}
+        if requested_rows:
+            # Explicit plugin retries are maintenance on the existing snapshot;
+            # preserve a sealed lifecycle while making its queued rows due.
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
             if preserve_queued and queued_rows:
-                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name in queued_rows}
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
                 affected_snapshot_ids.update(
                     str(snapshot_id)
                     for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
@@ -282,18 +313,12 @@ def run_plugins(
                     )
                 )
             requested_plugins_by_id: dict[str, set[str]] = defaultdict(set)
-            for snapshot_id, plugin_name in requested_pairs:
+            for snapshot_id, plugin_name, _hook_name in requested_rows:
                 requested_plugins_by_id[snapshot_id].add(plugin_name)
             for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
-                plugin_names = requested_plugins_by_id.get(str(snapshot.id), set())
-                sealed_search_backfill = (
-                    snapshot.status == Snapshot.StatusChoices.SEALED
-                    and plugin_names
-                    and all(plugin_name.startswith("search_backend_") for plugin_name in plugin_names)
-                )
-                if sealed_search_backfill:
+                if snapshot.status == Snapshot.StatusChoices.SEALED and requested_plugins_by_id.get(str(snapshot.id)):
                     snapshot.safe_update(
                         {"retry_at": queue_at, "modified_at": queue_at},
                         refresh=False,
@@ -318,7 +343,7 @@ def run_plugins(
                     refresh=False,
                     extra_filter={"status": snapshot.status},
                 )
-    if existing_crawl_ids and not requested_pairs:
+    if existing_crawl_ids and not requested_rows:
         from archivebox.crawls.models import Crawl
 
         for crawl in Crawl.objects.filter(id__in=existing_crawl_ids).only("id", "status", "retry_at", "modified_at"):
@@ -362,7 +387,7 @@ def run_plugins(
             selected_plugins = (
                 plugins_list
                 or sorted(
-                    {plugin for snapshot_id in crawl_snapshot_ids for plugin in requested_plugins_by_snapshot.get(str(snapshot_id), set())},
+                    {plugin for snapshot_id, plugin, _hook_name in requested_rows if snapshot_id in crawl_snapshot_ids},
                 )
                 or None
             )
@@ -371,6 +396,7 @@ def run_plugins(
                 snapshot_ids=sorted(crawl_snapshot_ids),
                 selected_plugins=selected_plugins,
                 show_progress=show_progress,
+                selected_plugins_are_explicit=bool(plugins_list),
             )
 
     if not emit_results:

archivebox/cli/archivebox_install.py

@@ -17,12 +17,12 @@ def _resolve_install_targets(
     requested_names: tuple[str, ...],
 ) -> tuple[list[str], list[str]]:
     """Resolve plugin names and declared binary aliases, leaving unknown tokens raw."""
-    from abx_dl.models import discover_plugins, filter_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
-    plugins = discover_plugins(runtime="archivebox")
-    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in plugins}
+    catalog = get_plugin_catalog()
+    plugin_names_by_lower = {plugin_name.lower(): plugin_name for plugin_name in catalog}
     plugin_names_by_binary_alias: dict[str, set[str]] = {}
-    for plugin_name, plugin in plugins.items():
+    for plugin_name, plugin in catalog.items():
         for required_binary in plugin.config.required_binaries:
             aliases = {required_binary.name.lower()}
             template_match = re.fullmatch(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", required_binary.name)
@@ -47,14 +47,14 @@ def _resolve_install_targets(
         else:
             raw_binary_names.append(name)
 
-    selected_plugins = filter_plugins(plugins, requested_plugins, include_providers=True) if requested_plugins else {}
+    selected_plugins = catalog.select(requested_plugins) if requested_plugins else catalog.select([])
     selected_plugin_names = {name.lower() for name in selected_plugins}
     raw_binary_names = [name for name in raw_binary_names if name.lower() not in selected_plugin_names]
     return sorted(selected_plugins), sorted(set(raw_binary_names))
 
 
 def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> None:
-    """Install user-requested standalone binaries through the Binary state machine."""
+    """Install user-requested standalone binaries through the Binary lifecycle."""
     from django.utils import timezone
 
     from archivebox.machine.models import Binary, Machine, _canonical_binary_name
@@ -179,7 +179,7 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
         run_install(plugin_names=install_plugin_names or None)
 
     if raw_binary_names:
-        print(f"[+] Running direct binary installer via ArchiveBox binary state machine: {', '.join(raw_binary_names)}")
+        print(f"[+] Running direct binary installer via ArchiveBox binary lifecycle: {', '.join(raw_binary_names)}")
         print()
         _install_raw_binary_names(raw_binary_names, binproviders)
 

archivebox/cli/archivebox_pluginmap.py

@@ -48,7 +48,7 @@ def pluginmap(
     Show the current abx-dl event phases and their associated plugin hooks.
 
     This command reflects the new bus-driven runtime, not the legacy ArchiveBox
-    state-machine executor. Event names are normalized to hook prefixes by
+    event runtime. Event names are normalized to hook prefixes by
     stripping a trailing `Event`, then ArchiveBox checks whether any matching
     `on_{EventFamily}__*` scripts actually exist.
     """

archivebox/cli/archivebox_run.py

@@ -156,18 +156,7 @@ def process_stdin_records() -> int:
                     queued_count += 1
 
             elif record_type == TYPE_ARCHIVERESULT:
-                if record_id:
-                    # Existing archiveresult - re-queue
-                    try:
-                        archiveresult = ArchiveResult.objects.get(id=record_id)
-                    except ArchiveResult.DoesNotExist:
-                        archiveresult = None
-                else:
-                    archiveresult = None
-
-                snapshot_id = record.get("snapshot_id")
-                plugin_name = record.get("plugin")
-                snapshot = None
+                archiveresult = ArchiveResult.from_json(record)
                 if archiveresult:
                     if archiveresult.status in [
                         ArchiveResult.StatusChoices.FAILED,
@@ -177,12 +166,10 @@ def process_stdin_records() -> int:
                     ]:
                         archiveresult.reset_for_retry()
                     snapshot = archiveresult.snapshot
-                    plugin_name = plugin_name or archiveresult.plugin
-                elif snapshot_id:
-                    try:
-                        snapshot = Snapshot.objects.get(id=snapshot_id)
-                    except Snapshot.DoesNotExist:
-                        snapshot = None
+                    plugin_name = archiveresult.plugin
+                else:
+                    snapshot = None
+                    plugin_name = None
 
                 if snapshot:
                     snapshot.queue_for_extraction()

archivebox/cli/archivebox_update.py

@@ -77,16 +77,16 @@ def reindex_snapshots(
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
     from archivebox.core.models import ArchiveResult, Snapshot
-    from abx_dl.models import discover_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
     # Search backfill is the one maintenance hook allowed to execute without
     # reopening a Snapshot. Restrict that exception to already-sealed rows;
-    # every open lifecycle state remains owned by the normal state machine.
+    # every open lifecycle state remains owned by the normal runner lifecycle.
     snapshots = snapshots.filter(status=Snapshot.StatusChoices.SEALED)
 
     stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
-    plugins_by_name = discover_plugins(runtime="archivebox")
+    plugins_by_name = get_plugin_catalog()
     required_hooks_by_plugin = {
         plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
         for plugin_name in search_plugins

archivebox/cli/archivebox_version.py

@@ -265,15 +265,14 @@ def version(
     seen_failures: set[str] = set()
     seen_rows: set[tuple[str, str, str, str]] = set()
 
-    from archivebox.plugins.discovery import get_enabled_plugins
+    from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from abx_dl.config import get_required_binary_requests
     from abx_dl.dependencies import resolve_binary_requests
-    from abx_dl.models import discover_plugins, filter_plugins
     from abx_dl.orchestrator import create_bus
     from abxpkg.binary_service import BinaryEvent, BinaryService
 
-    plugins = discover_plugins(runtime="archivebox")
-    enabled_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+    plugins = get_plugin_catalog()
+    enabled_plugins = plugins.select(get_enabled_plugins(config=config))
     enabled_plugin_names = set(enabled_plugins)
     runtime_config = normalize_runtime_config(config.for_crawl(), json_safe=False)
     derived_config: dict[str, object] = {}

archivebox/config/collection.py

@@ -160,7 +160,7 @@ def _coerce_from_str_dict(file_config: dict[str, str]) -> dict[str, Any]:
     so they round-trip through INI's string-only storage. When reading the
     file back into ``Machine.config`` (a JSONField that holds native types)
     those strings have to be decoded — otherwise downstream consumers like
-    ``_emit_machine_config`` → ``MachineEvent`` → abx-dl see a JSON string
+    ``ExecutionPlan.seed_config`` → ``MachineEvent`` → abx-dl see a JSON string
     where they expect a dict and raise ``TypeError``.
     Declared fields go through pydantic-settings' own ``field_is_complex`` /
     ``prepare_field_value`` so they're decoded per annotation. Undeclared

archivebox/config/common.py

@@ -16,7 +16,7 @@ from pathlib import Path
 from typing import Any, ClassVar, cast
 from urllib.parse import quote, urlparse
 
-from abx_plugins.plugins.base.utils import BASE_CONFIG_PATH, build_config_model, resolve_plugin_configs
+from abx_plugins.plugins.base.utils import build_config_model
 from django.db import DatabaseError
 from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -601,13 +601,9 @@ def _explicit_plugin_enabled_keys(config: Mapping[str, object]) -> set[str]:
 
 
 def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
-    from archivebox.plugins.discovery import discover_plugin_configs
+    from archivebox.plugins.discovery import get_plugin_config_resolver
 
-    schemas: PluginSchemaDocuments = {}
-    if BASE_CONFIG_PATH.exists():
-        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
-    schemas.update(discover_plugin_configs())
-    return schemas
+    return get_plugin_config_resolver().schemas
 
 
 def _plugin_config_properties(plugin_schemas: PluginSchemaDocuments) -> dict[str, dict[str, Any]]:
@@ -856,13 +852,11 @@ class ArchiveBoxBaseConfig(
             disabled_plugins = [plugin_name for plugin_name, enabled_key in enabled_config_keys.items() if not getattr(self, enabled_key)]
         selected_plugin_roots = plugin_names
         if selected_plugin_roots:
-            from abx_dl.models import discover_plugins, filter_plugins
+            from archivebox.plugins.discovery import get_plugin_catalog
 
             selected_plugins = set(
-                filter_plugins(
-                    discover_plugins(runtime="archivebox"),
+                get_plugin_catalog().select(
                     sorted(selected_plugin_roots),
-                    include_providers=True,
                     disabled_names=disabled_plugins,
                 ),
             )
@@ -1195,7 +1189,6 @@ def get_config(
 
     explicit_plugin_enabled_keys: set[str] = set()
     if resolve_plugins:
-        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
         crawl_selected_plugins = crawl_config_base and bool(_normalize_plugins_config_value(dict(crawl.config or {}).get("PLUGINS")))
         # A frozen crawl-level PLUGINS selector is the exact extractor set for
@@ -1219,8 +1212,9 @@ def get_config(
                 **_plugin_user_config(_plugin_input_config(file_config)),
                 **plugin_user_config,
             }
-        plugin_sections = resolve_plugin_configs(
-            plugin_schemas,
+        from archivebox.plugins.discovery import get_plugin_config_resolver
+
+        plugin_sections = get_plugin_config_resolver().resolve(
             global_config=plugin_global_config,
             user_config=plugin_user_config,
             environ={},

archivebox/core/admin_snapshots.py

@@ -346,19 +346,8 @@ class SnapshotAdminForm(forms.ModelForm):
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
-            if tags_str:
-                tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
-                tags = []
-                for name in tag_names:
-                    tag, _ = Tag.objects.get_or_create(
-                        name__iexact=name,
-                        defaults={"name": name},
-                    )
-                    tag = Tag.objects.filter(name__iexact=name).first() or tag
-                    tags.append(tag)
-                instance.tags.set(tags)
-            else:
-                instance.tags.clear()
+            tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+            instance.save_tags(tag_names)
 
         return instance
 
@@ -367,6 +356,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     form = SnapshotAdminForm
     raw_id_fields = ("crawl", "parent_snapshot")
     list_select_related = ()
+
+    def save_related(self, request, form, formsets, change):
+        super().save_related(request, form, formsets, change)
+        tags_str = form.cleaned_data.get("tags_editor", "")
+        tag_names = [name.strip() for name in tags_str.split(",") if name.strip()]
+        form.instance.save_tags(
+            tag_names,
+            created_by=request.user if request.user.is_authenticated else None,
+        )
+
     list_display = (
         "permissions_badge",
         "created_at",

archivebox/core/apps.py

@@ -10,7 +10,6 @@ class CoreConfig(AppConfig):
 
     def ready(self):
         """Register the archivebox.core.admin_site as the main django admin site"""
-        import sys
         from django.utils.autoreload import DJANGO_AUTORELOAD_ENV
 
         from archivebox.core.admin_site import register_admin_site
@@ -28,11 +27,6 @@ class CoreConfig(AppConfig):
 
         pre_save.connect(truncate_overlong_charfields, dispatch_uid="archivebox_truncate_overlong_charfields")
 
-        # Import models to register state machines with the registry
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.core import models  # noqa: F401
-
         def _should_prepare_runtime() -> bool:
             if os.environ.get("ARCHIVEBOX_RUNSERVER") == "1":
                 if os.environ.get("ARCHIVEBOX_AUTORELOAD") == "1":

archivebox/core/forms.py

@@ -12,9 +12,9 @@ from archivebox.core.widgets import TagEditorWidget, URLFiltersWidget
 from archivebox.crawls.schedule_util import validate_schedule
 from archivebox.misc.util import URL_REGEX, find_all_urls, parse_filesize_to_bytes
 from archivebox.personas.models import Persona
-from archivebox.plugins.discovery import get_plugins
+from archivebox.plugins.discovery import get_plugin_catalog
 from archivebox.plugins.forms import (
-    PLUGIN_GROUP_DEFINITIONS,
+    PLUGIN_GROUPS,
     TIMEOUT_INPUT_PATTERN,
     PluginConfigFormMixin,
     get_choice_field,
@@ -333,10 +333,9 @@ class AddLinkForm(PluginConfigFormMixin, forms.Form):
         if self.can_override_crawl_config:
             self.build_plugin_groups(get_config(persona=selected_persona) if selected_persona else get_config())
         else:
-            all_plugins = get_plugins()
-            for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
-                get_choice_field(self, field_name).choices = [(p, p) for p in all_plugins if p in plugin_names]
-            get_choice_field(self, "other_plugins").choices = [(p, p) for p in all_plugins]
+            grouped_plugins = get_plugin_catalog().groups()
+            for category, field_name, _title in PLUGIN_GROUPS:
+                get_choice_field(self, field_name).choices = [(plugin.name, plugin.name) for plugin in grouped_plugins.get(category, [])]
             self.plugin_groups = []
 
     def clean(self):

archivebox/core/migrations/0052_unique_archiveresult_per_snapshot_plugin.py

@@ -1,71 +0,0 @@
-from django.db import migrations, models
-from django.db.models import Sum
-
-
-def consolidate_archiveresults_per_plugin(apps, schema_editor):
-    ArchiveResult = apps.get_model("core", "ArchiveResult")
-    Snapshot = apps.get_model("core", "Snapshot")
-    duplicate_groups = ArchiveResult.objects.values("snapshot_id", "plugin").annotate(count=models.Count("id")).filter(count__gt=1)
-
-    affected_snapshot_ids = set()
-    for group in duplicate_groups.iterator(chunk_size=200):
-        rows = list(
-            ArchiveResult.objects.filter(
-                snapshot_id=group["snapshot_id"],
-                plugin=group["plugin"],
-            ).order_by("created_at", "id"),
-        )
-        winner = max(
-            rows,
-            key=lambda row: (
-                bool(row.output_files),
-                int(row.output_size or 0),
-                row.modified_at,
-                str(row.id),
-            ),
-        )
-        output_files = {}
-        mimetypes = set()
-        for row in rows:
-            output_files.update(row.output_files or {})
-            mimetypes.update(part.strip() for part in (row.output_mimetypes or "").split(",") if part.strip())
-
-        winner.output_files = output_files
-        winner.output_size = max(
-            sum(int(metadata.get("size") or 0) for metadata in output_files.values() if isinstance(metadata, dict)),
-            *(int(row.output_size or 0) for row in rows),
-        )
-        winner.output_mimetypes = ",".join(sorted(mimetypes))
-        winner.start_ts = min((row.start_ts for row in rows if row.start_ts), default=None)
-        winner.end_ts = max((row.end_ts for row in rows if row.end_ts), default=None)
-        winner.save(update_fields=["output_files", "output_size", "output_mimetypes", "start_ts", "end_ts"])
-        ArchiveResult.objects.filter(id__in=[row.id for row in rows if row.id != winner.id]).delete()
-        affected_snapshot_ids.add(group["snapshot_id"])
-
-    for snapshot_id in affected_snapshot_ids:
-        total = ArchiveResult.objects.filter(snapshot_id=snapshot_id).aggregate(total=Sum("output_size"))["total"] or 0
-        Snapshot.objects.filter(id=snapshot_id).update(output_size=total)
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("core", "0051_postgres_url_pattern_ops_index"),
-    ]
-
-    operations = [
-        migrations.RemoveConstraint(
-            model_name="archiveresult",
-            name="unique_archiveresult_per_snapshot_hook",
-        ),
-        migrations.RunPython(
-            consolidate_archiveresults_per_plugin,
-            reverse_code=migrations.RunPython.noop,
-        ),
-        migrations.AddConstraint(
-            model_name="archiveresult",
-            constraint=models.UniqueConstraint(
-                fields=("snapshot", "plugin"),
-                name="unique_archiveresult_per_snapshot_plugin",
-            ),
-        ),
-    ]

archivebox/core/migrations/0053_alter_archiveresult_options.py

@@ -5,7 +5,7 @@ from django.db import migrations
 
 class Migration(migrations.Migration):
     dependencies = [
-        ("core", "0052_unique_archiveresult_per_snapshot_plugin"),
+        ("core", "0051_postgres_url_pattern_ops_index"),
     ]
 
     operations = [

archivebox/core/recovery_util.py

@@ -10,10 +10,15 @@ def _is_signal_interrupted_exit(exit_code: int | None) -> bool:
     return exit_code is not None and (exit_code < 0 or exit_code >= 128)
 
 
+def _canonical_hook_name(hook_name: str) -> str:
+    hook_name = Path(hook_name).name
+    return Path(hook_name).stem if Path(hook_name).suffix in {".py", ".js", ".sh"} else hook_name
+
+
 def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str | None = None) -> dict[str, int]:
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.services.archive_result_service import _collect_output_metadata
+    from abx_dl.output_files import OutputManifest
     from archivebox.machine.models import Process
     from django.core.exceptions import ValidationError
     from django.db.models import Exists, OuterRef, Q, Subquery, Value
@@ -168,14 +173,23 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             orphaned_hook_processes = orphaned_hook_processes.filter(snapshot_pwd_filter)
         else:
             orphaned_hook_processes = orphaned_hook_processes.none()
-    for process in orphaned_hook_processes.only("id", "pwd", "cmd", "process_type", "status", "started_at", "ended_at").order_by(
-        "-started_at",
-        "-id",
-    ):
+    for process in orphaned_hook_processes.only(
+        "id",
+        "pwd",
+        "cmd",
+        "process_type",
+        "status",
+        "exit_code",
+        "stdout",
+        "stderr",
+        "started_at",
+        "ended_at",
+    ).order_by("-started_at", "-id"):
         hook_script_name = process.hook_script_name
         if not hook_script_name or not process.pwd:
             continue
         plugin_dir = Path(process.pwd)
+        hook_name = _canonical_hook_name(hook_script_name)
         if crawl_snapshot_ids is not None and plugin_dir.parent.name not in crawl_snapshot_ids:
             continue
         try:
@@ -187,29 +201,32 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
             continue
         if snapshot is None:
             continue
-        result, created = ArchiveResult.objects.get_or_create(
-            snapshot=snapshot,
-            plugin=plugin_dir.name,
+        result, created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            plugin_dir.name,
+            hook_name,
             defaults={
-                "hook_name": Path(hook_script_name).stem,
                 "status": ArchiveResult.StatusChoices.QUEUED,
             },
         )
-        if created or result.status == ArchiveResult.StatusChoices.QUEUED:
+        process_is_newer = bool(process.started_at and (result.start_ts is None or process.started_at >= result.start_ts))
+        if result.status == ArchiveResult.StatusChoices.QUEUED or process_is_newer:
             requeue_snapshot = False
             # A runner can die after the hook Process exits but before the
             # ProcessCompletedEvent projector links/finalizes ArchiveResult.
-            # Reconstruct the plugin row from its newest durable Process row.
-            output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+            # Reconstruct only that exact hook row from the durable Process row.
+            manifest = OutputManifest.scan(plugin_dir, containment_root=snapshot.output_dir)
+            output_files = manifest.as_mapping()
+            output_size = manifest.total_size
+            output_mimetypes = ",".join(manifest.mimetypes)
             emitted_records = [
                 record
                 for record in Process.parse_records_from_text(process.stdout or "")
                 if record.get("type") == "ArchiveResult"
                 and (record.get("plugin") or plugin_dir.name) == plugin_dir.name
-                and (record.get("hook_name") or Path(hook_script_name).stem) == Path(hook_script_name).stem
+                and _canonical_hook_name(str(record.get("hook_name") or hook_name)) == hook_name
             ]
             emitted_result = emitted_records[-1] if emitted_records else {}
-            result.hook_name = Path(hook_script_name).stem
             result.process = process
             result.start_ts = process.started_at
             result.end_ts = process.ended_at
@@ -222,6 +239,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 result.output_size = 0
                 result.output_mimetypes = ""
                 result.output_str = ""
+                result.output_json = None
                 result.status = ArchiveResult.StatusChoices.QUEUED
                 requeue_snapshot = True
             else:
@@ -246,7 +264,6 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
                 )
             result.save(
                 update_fields=[
-                    "hook_name",
                     "process",
                     "start_ts",
                     "end_ts",
@@ -287,7 +304,7 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
 
     # Broken lock repair: STARTED + retry_at=NULL is an orphaned ownership
     # lease. Recovery only unlocks scheduling; the runner owns any subsequent
-    # state-machine transition, including sealing rows whose children/results
+    # lifecycle transition, including sealing rows whose children/results
     # are already final.
     recoverable_started_crawls = Crawl.objects.filter(status=Crawl.StatusChoices.STARTED).filter(
         Q(retry_at__isnull=True) | Q(retry_at__gt=now),

archivebox/core/tag_util.py

@@ -152,15 +152,8 @@ def get_or_create_tag(name: str, created_by: User | None = None) -> tuple[Tag, b
     if not normalized_name:
         raise ValueError("Tag name is required")
 
-    existing = Tag.objects.filter(name__iexact=normalized_name).first()
-    if existing:
-        return existing, False
-
-    tag = Tag.objects.create(
-        name=normalized_name,
-        created_by=created_by,
-    )
-    return tag, True
+    defaults = {"created_by": created_by} if created_by is not None else None
+    return Tag.get_or_create_by_name(normalized_name, defaults=defaults)
 
 
 def rename_tag(tag: Tag, name: str) -> Tag:

archivebox/core/templatetags/core_tags.py

@@ -32,7 +32,7 @@ _TEXT_PREVIEW_EXTS = (".json", ".jsonl", ".txt", ".csv", ".tsv", ".xml", ".yml",
 _IMAGE_PREVIEW_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".ico", ".avif")
 _STATIC_URL_SAFE = "/@-._~!$&'()*+,;="
 
-_MEDIA_FILE_EXTS = {
+_VIDEO_FILE_EXTS = {
     ".mp4",
     ".webm",
     ".mkv",
@@ -49,6 +49,9 @@ _MEDIA_FILE_EXTS = {
     ".3gp",
     ".3g2",
     ".ogv",
+}
+
+_AUDIO_FILE_EXTS = {
     ".mp3",
     ".m4a",
     ".aac",
@@ -66,16 +69,15 @@ _MEDIA_FILE_EXTS = {
     ".dts",
 }
 
+_MEDIA_FILE_EXTS = _VIDEO_FILE_EXTS | _AUDIO_FILE_EXTS
+_BROWSER_VIDEO_FILE_EXTS = {".mp4", ".webm", ".m4v", ".ogv"}
+_BROWSER_AUDIO_FILE_EXTS = {".mp3", ".m4a", ".aac", ".ogg", ".oga", ".opus", ".wav", ".flac"}
+
 
 def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
-    if isinstance(output_files, dict):
-        normalized: dict[str, dict[str, Any]] = {}
-        for path, metadata in output_files.items():
-            if not path:
-                continue
-            normalized[str(path)] = dict(metadata) if isinstance(metadata, dict) else {}
-        return normalized
-    return {}
+    from abx_dl.output_files import OutputManifest
+
+    return OutputManifest.from_value(output_files).as_mapping()
 
 
 def _snapshot_id(value: Any) -> Any:
@@ -159,15 +161,27 @@ def _list_media_files(result) -> list[dict]:
 
     for rel_path, size in candidates:
         href = str(Path(result.plugin) / rel_path)
+        suffix = rel_path.suffix.lower()
+        media_type = "video" if suffix in _VIDEO_FILE_EXTS else "audio"
         media_files.append(
             {
                 "name": rel_path.name,
                 "path": href,
                 "size": size,
+                "media_type": media_type,
+                "is_video": media_type == "video",
+                "is_audio": media_type == "audio",
+                "is_browser_playable": suffix in _BROWSER_VIDEO_FILE_EXTS or suffix in _BROWSER_AUDIO_FILE_EXTS,
             },
         )
 
-    media_files.sort(key=lambda item: item["name"].lower())
+    media_files.sort(
+        key=lambda item: (
+            0 if item["is_video"] and item["is_browser_playable"] else 1 if item["is_audio"] and item["is_browser_playable"] else 2,
+            -int(item.get("size") or 0),
+            item["name"].lower(),
+        ),
+    )
     return media_files
 
 

archivebox/core/views.py

@@ -691,26 +691,27 @@ def _resolve_archiveresult_relpath(snapshot: Snapshot, rel_path: str) -> tuple[s
 
     plugin = parts[0]
     plugin_relpath = posixpath.join(*parts[1:])
-    result = (
-        ArchiveResult.objects.filter(snapshot=snapshot, plugin=plugin, status=ArchiveResult.StatusChoices.SUCCEEDED)
-        .only("plugin", "output_files")
-        .first()
+    results = list(
+        ArchiveResult.objects.filter(
+            snapshot=snapshot,
+            plugin=plugin,
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+        ).only("plugin", "output_files"),
     )
-    if not result:
+    if not results:
         return rel_path, None
-    if not result.output_files:
-        return rel_path, result
 
-    output_files = result.output_files or {}
-    for candidate in (plugin_relpath, rel_path):
-        file_info = output_files.get(candidate)
-        if not isinstance(file_info, dict):
-            continue
-        if file_info.get("root_relative"):
-            return candidate, result
-        return rel_path, result
+    for result in results:
+        output_files = result.output_files or {}
+        for candidate in (plugin_relpath, rel_path):
+            file_info = output_files.get(candidate)
+            if not isinstance(file_info, dict):
+                continue
+            if file_info.get("root_relative"):
+                return candidate, result
+            return rel_path, result
 
-    return rel_path, result
+    return rel_path, results[0]
 
 
 def _plugin_full_preview_response(

archivebox/crawls/apps.py

@@ -5,11 +5,3 @@ class CrawlsConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "archivebox.crawls"
     label = "crawls"
-
-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.crawls.models import CrawlMachine  # noqa: F401

archivebox/crawls/models.py

@@ -19,7 +19,6 @@ from django.core.validators import MaxValueValidator, MinValueValidator
 from django.conf import settings
 from django.urls import reverse_lazy
 from django.utils import timezone
-from statemachine import State, registry
 from archivebox.config.common import rprint as print
 from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
 
@@ -32,7 +31,7 @@ from archivebox.base_models.models import (
     ModelWithHealthStats,
     get_or_create_system_user_pk,
 )
-from archivebox.workers.models import RETRY_AT_MAX, ModelWithStateMachine, BaseStateMachine
+from archivebox.workers.models import ModelWithQueue
 from archivebox.crawls.schedule_util import next_run_for_schedule, validate_schedule
 from archivebox.misc.util import parse_date, sanitize_html_text, validate_url, validate_url_length
 
@@ -129,7 +128,7 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         )
 
 
-class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithStateMachine):
+class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
@@ -158,16 +157,19 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     notes = models.TextField(blank=True, null=False, default="")
     schedule = models.ForeignKey(CrawlSchedule, on_delete=models.SET_NULL, null=True, blank=True, editable=True)
 
-    status = ModelWithStateMachine.StatusField(
-        choices=ModelWithStateMachine.StatusChoices,
-        default=ModelWithStateMachine.StatusChoices.QUEUED,
+    status = ModelWithQueue.StatusField(
+        choices=ModelWithQueue.StatusChoices,
+        default=ModelWithQueue.StatusChoices.QUEUED,
     )
-    retry_at = ModelWithStateMachine.RetryAtField(default=timezone.now)
+    retry_at = ModelWithQueue.RetryAtField(default=timezone.now)
 
-    state_machine_name = "archivebox.crawls.models.CrawlMachine"
     retry_at_field_name = "retry_at"
     state_field_name = "status"
-    StatusChoices = ModelWithStateMachine.StatusChoices
+    StatusChoices = ModelWithQueue.StatusChoices
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.STARTED
+    FINAL_STATES = (StatusChoices.SEALED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
     active_state = StatusChoices.STARTED
     delete_after_final_statuses = (StatusChoices.SEALED,)
     RUNNABLE_STATES = (StatusChoices.QUEUED, StatusChoices.STARTED)
@@ -177,17 +179,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
     snapshot_set: models.Manager["Snapshot"]
 
-    if TYPE_CHECKING:
-
-        @property
-        def sm(self) -> "CrawlMachine": ...
-
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
         ModelWithConfig.Meta,
         ModelWithHealthStats.Meta,
-        ModelWithStateMachine.Meta,
+        ModelWithQueue.Meta,
     ):
         app_label = "crawls"
         verbose_name = "Crawl"
@@ -255,7 +252,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         now = timezone.now()
         # Cancellation seals the Crawl first, then lets the runner seal each
-        # child Snapshot through its own state machine. Active children that
+        # child Snapshot through its own lifecycle. Active children that
         # are already due need no write; the runner will claim them as-is.
         active_children = self.snapshot_set.filter(
             status__in=Snapshot.OPEN_STATES,
@@ -1194,7 +1191,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 ),
             }
             if tag_names:
-                tag_ids = [Tag.objects.get_or_create(name=tag_name)[0].pk for tag_name in tag_names]
+                tag_ids = [Tag.get_or_create_by_name(tag_name)[0].pk for tag_name in tag_names]
                 snapshot.add_tag_ids(tag_ids)
 
         existing_scope = Snapshot.objects if bool(self._config_value(config, "ONLY_NEW", True)) else self.snapshot_set
@@ -1296,170 +1293,6 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return created_snapshots
 
-    def install_declared_binaries(self, binary_names: set[str], machine=None) -> None:
-        """Install crawl-declared binaries through their unified state machine."""
-        from archivebox.crawls.locks import binary_lifecycle_lock
-        from archivebox.machine.models import Binary, Machine
-
-        if not binary_names:
-            return
-
-        machine = machine or Machine.current()
-        binaries = Binary.objects.filter(machine=machine, name__in=binary_names).order_by("name")
-        for binary in binaries:
-            with binary_lifecycle_lock(str(binary.id)):
-                binary.refresh_from_db()
-                if binary.status == Binary.StatusChoices.INSTALLED:
-                    continue
-                binary.update_and_requeue(retry_at=timezone.now())
-                binary.refresh_from_db()
-                binary.tick_claimed(lock_seconds=600)
-
-        unresolved_binaries = list(
-            Binary.objects.filter(
-                machine=machine,
-                name__in=binary_names,
-            )
-            .exclude(
-                status=Binary.StatusChoices.INSTALLED,
-            )
-            .order_by("name"),
-        )
-        if unresolved_binaries:
-            binary_details = ", ".join(
-                f"{binary.name} (status={binary.status}, retry_at={binary.retry_at})" for binary in unresolved_binaries
-            )
-            raise RuntimeError(
-                f"Crawl dependencies failed to install before continuing: {binary_details}",
-            )
-
-    def run(self) -> "Snapshot | None":
-        """
-        Execute this Crawl: run hooks, process JSONL, create snapshots.
-
-        Called by the state machine when entering the 'started' state.
-
-        Returns:
-            The root Snapshot for this crawl, or None for system crawls that don't create snapshots
-        """
-        import time
-        from archivebox.plugins.hooks import run_hook, discover_hooks, process_hook_records
-        from archivebox.config.common import get_config
-        from archivebox.machine.models import Machine
-
-        def get_runtime_config():
-            return get_config(crawl=self).for_crawl_runtime(
-                crawl=self,
-                persona=persona,
-                runtime_overrides=persona_runtime_overrides,
-            )
-
-        system_task = self.get_system_task()
-        if system_task == "archivebox://update":
-            from archivebox.cli.archivebox_update import process_all_db_snapshots
-
-            process_all_db_snapshots()
-            return None
-
-        machine = Machine.current()
-        declared_binary_names: set[str] = set()
-        persona_runtime_overrides: dict[str, str] = {}
-        persona = self.resolve_persona()
-        if persona:
-            base_runtime_config = get_config(crawl=self, persona=persona)
-            chrome_binary = str(base_runtime_config.get("CHROME_BINARY") or "")
-            persona_runtime_overrides = persona.prepare_runtime_for_crawl(
-                crawl=self,
-                chrome_binary=chrome_binary,
-            )
-
-        def run_crawl_hook(hook: Path) -> set[str]:
-            primary_url = next(
-                (line.strip() for line in self.urls.splitlines() if line.strip()),
-                self.urls.strip(),
-            )
-
-            hook_start = time.time()
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=get_runtime_config(),
-                crawl_id=str(self.id),
-                source_url=self.urls,
-                url=primary_url,
-                snapshot_id=str(self.id),
-            )
-            hook_elapsed = time.time() - hook_start
-            if hook_elapsed > 0.5:
-                print(f"[yellow]⏱️  Hook {hook.name} took {hook_elapsed:.2f}s[/yellow]")
-
-            if process.status == process.StatusChoices.RUNNING:
-                if process.poll() is None:
-                    return set()
-
-            from archivebox.plugins.hooks import extract_records_from_process
-
-            records = []
-            # A hook can exit before its completed Process metadata is visible.
-            # Give successful hooks a brief chance to flush JSONL stdout into
-            # the Process row before downstream hooks.
-            for delay in (0.0, 0.05, 0.1, 0.25, 0.5):
-                if delay:
-                    time.sleep(delay)
-                records = extract_records_from_process(process)
-                if records:
-                    break
-            if records:
-                print(f"[cyan]📝 Processing {len(records)} records from {hook.name}[/cyan]")
-                for record in records[:3]:
-                    print(f"   Record: type={record.get('type')}, keys={list(record.keys())[:5]}")
-            if system_task:
-                records = [record for record in records if record.get("type") in ("BinaryRequest", "Binary")]
-            overrides = {"crawl": self}
-            stats = process_hook_records(records, overrides=overrides)
-            if stats:
-                print(f"[green]✓ Created: {stats}[/green]")
-
-            hook_binary_names = {
-                str(record.get("name")).strip()
-                for record in records
-                if record.get("type") in ("BinaryRequest", "Binary") and record.get("name")
-            }
-            hook_binary_names.discard("")
-            if hook_binary_names:
-                declared_binary_names.update(hook_binary_names)
-            return hook_binary_names
-
-        hooks = discover_hooks("Crawl", config=get_runtime_config())
-
-        for hook in hooks:
-            hook_binary_names = run_crawl_hook(hook)
-            if hook_binary_names:
-                self.install_declared_binaries(hook_binary_names, machine=machine)
-
-        # Safety check: don't create snapshots if any crawl-declared dependency
-        # is still unresolved after all crawl hooks have run.
-        self.install_declared_binaries(declared_binary_names, machine=machine)
-
-        # Create snapshots from all URLs in self.urls
-        if system_task:
-            leaked_snapshots = self.snapshot_set.all()
-            if leaked_snapshots.exists():
-                leaked_count = leaked_snapshots.count()
-                leaked_snapshots.delete()
-                print(f"[yellow]⚠️  Removed {leaked_count} leaked snapshot(s) created during system crawl {system_task}[/yellow]")
-            return None
-
-        self.create_snapshots_from_urls()
-
-        # Return first snapshot for this crawl (newly created or existing)
-        # This ensures the crawl doesn't seal if snapshots exist, even if they weren't just created
-        return self.snapshot_set.first()
-
     def is_finished(self) -> bool:
         """Check if crawl is finished (all snapshots sealed or no snapshots exist)."""
         from archivebox.core.models import Snapshot
@@ -1483,11 +1316,64 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
 
         return True
 
-    def cleanup(self):
-        """Clean up background hooks and run on_CrawlEnd hooks."""
-        from archivebox.plugins.hooks import run_hook, discover_hooks
+    def can_start(self) -> bool:
+        return bool(self.urls and self.get_urls_list())
 
-        # Clean up .pid files from output directory
+    def has_finished_snapshots(self) -> bool:
+        from archivebox.core.models import Snapshot
+
+        snapshots = self.snapshot_set.all()
+        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
+
+    def mark_started(self) -> bool:
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.STARTED,
+                "retry_at": now + timedelta(seconds=2),
+            },
+            extra_filter={"status": self.StatusChoices.QUEUED},
+        )
+        return updated
+
+    def seal(self) -> bool:
+        """Finalize a runner-owned Crawl without dispatching hooks directly."""
+        now = timezone.now()
+        updated = self.safe_update(
+            {
+                "status": self.StatusChoices.SEALED,
+                "retry_at": None,
+                "modified_at": now,
+            },
+            refresh=False,
+            extra_filter={"status__in": (*self.RUNNABLE_STATES, self.StatusChoices.SEALED)},
+        )
+        if not updated:
+            self.refresh_from_db()
+            return False
+        self.status = self.StatusChoices.SEALED
+        self.retry_at = None
+        self.modified_at = now
+        self.schedule_child_snapshots_for_sealing()
+        self.cleanup_runtime()
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance one explicit lifecycle step after the runner claims this row."""
+        if self.status == self.StatusChoices.PAUSED:
+            return False
+        if self.status == self.StatusChoices.QUEUED:
+            if self.has_finished_snapshots():
+                return self.seal()
+            if not self.can_start():
+                return False
+            return self.mark_started()
+        if self.status == self.StatusChoices.STARTED and self.is_finished():
+            return self.seal()
+        return False
+
+    def cleanup_runtime(self) -> None:
+        """Remove runner-owned runtime artifacts after abx-dl cleanup hooks finish."""
         if self.output_dir.exists():
             for pid_file in self.output_dir.glob("**/*.pid"):
                 pid_file.unlink(missing_ok=True)
@@ -1495,207 +1381,3 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         persona = self.resolve_persona()
         if persona:
             persona.cleanup_runtime_for_crawl(self)
-
-        # Run on_CrawlEnd hooks
-        from archivebox.config.common import get_config
-
-        config = get_config(crawl=self)
-
-        hooks = discover_hooks("CrawlEnd", config=config)
-
-        for hook in hooks:
-            plugin_name = hook.parent.name
-            output_dir = self.output_dir / plugin_name
-            output_dir.mkdir(parents=True, exist_ok=True)
-
-            process = run_hook(
-                hook,
-                output_dir=output_dir,
-                config=config,
-                crawl_id=str(self.id),
-                source_url=self.urls,  # Pass full newline-separated URLs
-            )
-
-            # Log failures but don't block
-            if process.exit_code != 0:
-                print(f"[yellow]⚠️ CrawlEnd hook failed: {hook.name}[/yellow]")
-
-
-# =============================================================================
-# State Machines
-# =============================================================================
-
-
-class CrawlMachine(BaseStateMachine):
-    crawl: Crawl
-
-    """
-    State machine for managing Crawl lifecycle.
-
-    Hook Lifecycle:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Waiting for crawl to be ready (has URLs)                 │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_start()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ STARTED State → enter_started()                             │
-    │  1. crawl.run()                                             │
-    │     • discover_hooks('Crawl') → finds all crawl hooks       │
-    │     • For each hook:                                        │
-    │       - run_hook(script, output_dir, ...)                   │
-    │       - Parse JSONL from hook output                        │
-    │       - process_hook_records() → creates Snapshots          │
-    │     • create_snapshots_from_urls() → from self.urls field   │
-    │                                                              │
-    │  2. Snapshots process independently with their own          │
-    │     state machines (see SnapshotMachine)                    │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when is_finished()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ SEALED State → enter_sealed()                               │
-    │  • cleanup() → runs on_CrawlEnd hooks, kills background     │
-    │  • Set retry_at=None (no more processing)                   │
-    └─────────────────────────────────────────────────────────────┘
-    """
-
-    model_attr_name = "crawl"
-
-    # States
-    queued = State(value=Crawl.StatusChoices.QUEUED, initial=True)
-    started = State(value=Crawl.StatusChoices.STARTED)
-    paused = State(value=Crawl.StatusChoices.PAUSED)
-    sealed = State(value=Crawl.StatusChoices.SEALED, final=True)
-
-    # Tick Event (polled by workers)
-    tick = (
-        queued.to(sealed, cond="has_finished_snapshots")
-        | queued.to.itself(unless="can_start")
-        | queued.to(started, cond="can_start")
-        | started.to(sealed, cond="is_finished")
-        | paused.to.itself()
-    )
-
-    # Manual event (triggered by last Snapshot sealing, or by direct
-    # index-only/bg creation when every requested URL is rejected before any
-    # Snapshot rows exist).
-    seal = queued.to(sealed) | started.to(sealed) | paused.to(sealed)
-    pause_requested = queued.to(paused) | started.to(paused)
-    resume_requested = paused.to(queued)
-
-    def can_start(self) -> bool:
-        if not self.crawl.urls:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no URLs[/red]")
-            return False
-        urls_list = self.crawl.get_urls_list()
-        if not urls_list:
-            print(f"[red]⚠️ Crawl {self.crawl.id} cannot start: no valid URLs in urls field[/red]")
-            return False
-        return True
-
-    def is_finished(self) -> bool:
-        """Check if all Snapshots for this crawl are finished."""
-        return self.crawl.is_finished()
-
-    def has_finished_snapshots(self) -> bool:
-        """A queued crawl with only final Snapshot rows was interrupted before sealing."""
-        from archivebox.core.models import Snapshot
-
-        snapshots = self.crawl.snapshot_set.all()
-        return snapshots.exists() and not snapshots.exclude(status=Snapshot.StatusChoices.SEALED).exists()
-
-    @queued.enter
-    def enter_queued(self):
-        self.crawl.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Crawl.StatusChoices.QUEUED,
-        )
-
-    @started.enter
-    def enter_started(self):
-        import sys
-
-        print(f"[cyan]🔄 CrawlMachine.enter_started() - creating snapshots for {self.crawl.id}[/cyan]", file=sys.stderr)
-
-        try:
-            # Run the crawl - runs hooks, processes JSONL, creates snapshots
-            first_snapshot = self.crawl.run()
-
-            if first_snapshot:
-                print(
-                    f"[cyan]🔄 Created {self.crawl.snapshot_set.count()} snapshot(s), first: {first_snapshot.url}[/cyan]",
-                    file=sys.stderr,
-                )
-                # Update status to STARTED
-                # Set retry_at to near future so tick() can poll and check is_finished()
-                self.crawl.update_and_requeue(
-                    retry_at=timezone.now() + timedelta(seconds=2),
-                    status=Crawl.StatusChoices.STARTED,
-                )
-            else:
-                # No snapshots (system crawl that only runs setup hooks)
-                print("[cyan]🔄 No snapshots created, sealing crawl immediately[/cyan]", file=sys.stderr)
-                # Seal immediately since there's no work to do
-                self.seal()
-
-        except Exception as e:
-            print(f"[red]⚠️ Crawl {self.crawl.id} failed to start: {e}[/red]")
-            import traceback
-
-            traceback.print_exc()
-            raise
-
-    @paused.enter
-    def enter_paused(self):
-        paused = self.crawl.safe_update(
-            {
-                "retry_at": RETRY_AT_MAX,
-                "status": Crawl.StatusChoices.PAUSED,
-            },
-            extra_filter={"status__in": Crawl.RUNNABLE_STATES},
-        )
-        if paused:
-            self.crawl.schedule_child_snapshots_for_pause()
-
-    @sealed.enter
-    def enter_sealed(self):
-        now = timezone.now()
-        self.crawl.status = Crawl.StatusChoices.SEALED
-        self.crawl.retry_at = None
-        # Guard: never seal a row that a concurrent writer flipped to PAUSED.
-        # Sealing is idempotent (SEALED→SEALED is a no-op rewrite), so
-        # status__in covers both the QUEUED/STARTED→SEALED transition and the
-        # rare re-entry case.
-        updated = self.crawl.safe_update(
-            {
-                "status": Crawl.StatusChoices.SEALED,
-                "retry_at": None,
-                "modified_at": now,
-            },
-            refresh=False,
-            extra_filter={
-                "status__in": [
-                    Crawl.StatusChoices.QUEUED,
-                    Crawl.StatusChoices.STARTED,
-                    Crawl.StatusChoices.SEALED,
-                ],
-            },
-        )
-        if not updated:
-            self.crawl.refresh_from_db()
-            return
-        self.crawl.modified_at = now
-
-        self.crawl.schedule_child_snapshots_for_sealing()
-        # Clean up background hooks and run on_CrawlEnd hooks after the final
-        # state is visible so cleanup projectors cannot resurrect the crawl.
-        self.crawl.cleanup()
-
-
-# =============================================================================
-# Register State Machines
-# =============================================================================
-
-# Manually register state machines with python-statemachine registry
-# (normally auto-discovered from statemachines.py, but we define them here for clarity)
-registry.register(CrawlMachine)

archivebox/machine/apps.py

@@ -10,14 +10,6 @@ class MachineConfig(AppConfig):
     label = "machine"  # Explicit label for migrations
     verbose_name = "Machine Info"
 
-    def ready(self):
-        """Import models to register state machines with the registry"""
-        import sys
-
-        # Skip during makemigrations to avoid premature state machine access
-        if "makemigrations" not in sys.argv:
-            from archivebox.machine import models  # noqa: F401
-
 
 def register_admin(admin_site):
     from archivebox.machine.admin import register_admin

archivebox/machine/models.py

@@ -12,8 +12,6 @@ from archivebox.uuid_compat import CompactUUIDField, uuid7
 from datetime import timedelta, datetime
 from typing import TYPE_CHECKING, Any, cast
 
-from statemachine import State, registry
-
 from django.db import IntegrityError, transaction
 from django.db import models
 from django.db.models import Q, QuerySet
@@ -23,7 +21,7 @@ from django.utils.functional import cached_property
 from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
 from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
-from archivebox.workers.models import BaseStateMachine, ModelWithStateMachine
+from archivebox.workers.models import ModelWithQueue
 from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
 
 _psutil: Any | None = None
@@ -515,19 +513,21 @@ class BinaryManager(models.Manager):
         )
 
 
-class Binary(ModelWithHealthStats, ModelWithStateMachine):
+class Binary(ModelWithHealthStats, ModelWithQueue):
     """
     Tracks a binary on a specific machine.
 
-    Simple state machine with 2 states:
+    Simple queue lifecycle with 2 states:
     - queued: Binary needs to be installed
     - installed: Binary installed successfully (abspath, version, sha256 populated)
 
     Installation is synchronous during queued→installed transition.
     If installation fails, Binary stays in queued with retry_at set for later retry.
 
-    State machine calls run(), which emits an abxpkg BinaryRequestEvent through
-    the ArchiveBox runner and installs the binary using the specified providers.
+    BinaryService claims queued rows with a conditional update, then run()
+    emits an abxpkg BinaryRequestEvent and persists the resolved installation.
+    The database row is the only lifecycle state; there is intentionally no
+    second in-memory state machine to reconcile after a worker interruption.
     """
 
     class StatusChoices(models.TextChoices):
@@ -566,9 +566,9 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
     version = models.CharField(max_length=32, default="", null=False, blank=True)
     sha256 = models.CharField(max_length=64, default="", null=False, blank=True)
 
-    # State machine fields
-    status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
-    retry_at = ModelWithStateMachine.RetryAtField(
+    # Durable queue lifecycle fields
+    status = ModelWithQueue.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED, max_length=16)
+    retry_at = ModelWithQueue.RetryAtField(
         default=timezone.now,
         help_text="When to retry this binary installation",
     )
@@ -579,18 +579,16 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
     machine_id: uuid.UUID
 
-    state_machine_name: str | None = "archivebox.machine.models.BinaryMachine"
+    INITIAL_STATE = StatusChoices.QUEUED
+    ACTIVE_STATE = StatusChoices.QUEUED
+    FINAL_STATES = (StatusChoices.INSTALLED,)
+    FINAL_OR_ACTIVE_STATES = (*FINAL_STATES, ACTIVE_STATE)
     active_state: str = StatusChoices.QUEUED
     warn_on_save_outside_runner = False
 
     objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]
 
-    if TYPE_CHECKING:
-
-        @property
-        def sm(self) -> BinaryMachine: ...
-
-    class Meta(ModelWithHealthStats.Meta, ModelWithStateMachine.Meta):
+    class Meta(ModelWithHealthStats.Meta, ModelWithQueue.Meta):
         app_label = "machine"
         verbose_name = "Binary"
         verbose_name_plural = "Binaries"
@@ -604,6 +602,10 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         """A binary is valid if it has a resolved path and is marked installed."""
         return bool(self.abspath) and self.status == self.StatusChoices.INSTALLED
 
+    @property
+    def can_install(self) -> bool:
+        return bool(self.name and self.binproviders)
+
     @cached_property
     def binary_info(self) -> dict:
         """Return info about the binary."""
@@ -748,11 +750,43 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
         run_binary(str(self.id))
 
+    def install(self) -> bool:
+        """Run one synchronous installation attempt for a claimed Binary."""
+        if self.status == self.StatusChoices.INSTALLED:
+            return True
+        if not self.can_install:
+            return False
+
+        rprint(f"[cyan]      🔄 installing {self.name}[/cyan]", file=sys.stderr)
+        self.run()
+        self.refresh_from_db()
+        if self.status != self.StatusChoices.INSTALLED:
+            self.update_and_requeue(
+                retry_at=timezone.now() + timedelta(seconds=300),
+                status=self.StatusChoices.QUEUED,
+            )
+            self.increment_health_stats(success=False)
+            raise RuntimeError(f"Binary {self.name} installation failed")
+
+        self.update_and_requeue(retry_at=None, status=self.StatusChoices.INSTALLED)
+        self.increment_health_stats(success=True)
+        return True
+
+    def advance_lifecycle(self) -> bool:
+        """Advance the explicit binary lifecycle after its queue row is claimed."""
+        return self.install()
+
+    def install_claimed(self, *, lock_seconds: int = 600) -> bool:
+        if not self.claim_processing_lock(lock_seconds=lock_seconds):
+            return False
+        self.refresh_from_db()
+        return self.advance_lifecycle()
+
     def cleanup(self):
         """
         Clean up background binary installation hooks.
 
-        Called by state machine if needed (not typically used for binaries
+        Called after an installation attempt if needed (not typically used for binaries
         since installations are foreground, but included for consistency).
         """
 
@@ -953,12 +987,14 @@ class Process(ModelWithDeleteAfter, models.Model):
     One Process can optionally be associated with an ArchiveResult (via OneToOne),
     but Process can also exist standalone for internal operations.
 
-    Follows the unified state machine pattern:
+    Follows the unified process lifecycle:
     - queued: Process ready to launch
     - running: Process actively executing
     - exited: Process completed (check exit_code for success/failure)
 
-    State machine calls launch() to spawn the process and monitors its lifecycle.
+    Direct Process methods and the abx-dl event projector own these transitions.
+    Keeping the DB row as the only lifecycle state makes interrupted subprocess
+    recovery observable without reconciling a second in-memory state machine.
     """
 
     class StatusChoices(models.TextChoices):
@@ -1117,7 +1153,7 @@ class Process(ModelWithDeleteAfter, models.Model):
     # Reverse relation to ArchiveResult (OneToOne from AR side)
     # archiveresult: OneToOneField defined on ArchiveResult model
 
-    # State machine fields
+    # Durable process lifecycle fields
     status = models.CharField(
         max_length=16,
         choices=StatusChoices.choices,
@@ -1138,7 +1174,6 @@ class Process(ModelWithDeleteAfter, models.Model):
     children: models.Manager[Process]
     archiveresult: ArchiveResult
 
-    state_machine_name: str = "archivebox.machine.models.ProcessMachine"
     delete_after_final_statuses = (StatusChoices.EXITED,)
 
     objects = ProcessManager()  # pyright: ignore[reportIncompatibleVariableOverride]
@@ -1308,8 +1343,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         Compare-and-swap update for short Process scheduler writes.
 
-        Process is not a ModelWithStateMachine subclass yet, but its
-        state-machine methods still need the same modified_at CAS behavior as
+        Process is not a ModelWithQueue subclass, but its scheduler methods
+        still need the same modified_at CAS behavior as
         Crawl/Snapshot/Binary without falling back to save().
         """
         values = dict(update_fields)
@@ -2233,8 +2268,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         Gracefully terminate process: SIGTERM → wait → SIGKILL.
 
-        This consolidates the scattered SIGTERM/SIGKILL logic from:
-        - crawls/models.py Crawl.cleanup()
+        This consolidates SIGTERM/SIGKILL logic used by:
+        - workers/management/commands/runner_watch.py
         - workers/pid_utils.py stop_worker()
         - supervisord_util.py stop_existing_supervisord_process()
 
@@ -2297,8 +2332,8 @@ class Process(ModelWithDeleteAfter, models.Model):
         Uses parallel polling approach - sends SIGTERM to all processes at once,
         then polls all simultaneously with individual deadline tracking.
 
-        This consolidates the scattered child-killing logic from:
-        - crawls/models.py Crawl.cleanup() os.killpg()
+        This consolidates child-killing logic used by:
+        - core/takeover_util.py
         - supervisord_util.py stop_existing_supervisord_process()
 
         Args:
@@ -2598,206 +2633,3 @@ class Process(ModelWithDeleteAfter, models.Model):
         if cleaned:
             rprint(f"[yellow]🧹 Cleaned up {cleaned} orphaned worker/hook process record(s)[/yellow]")
         return cleaned
-
-
-# =============================================================================
-# Binary State Machine
-# =============================================================================
-
-
-class BinaryMachine(BaseStateMachine):
-    """
-    State machine for managing Binary installation lifecycle.
-
-    Simple 2-state machine:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Binary needs to be installed                             │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_install()
-                            ↓ Synchronous installation during transition
-    ┌─────────────────────────────────────────────────────────────┐
-    │ INSTALLED State                                             │
-    │  • Binary installed (abspath, version, sha256 set)          │
-    │  • Health stats incremented                                 │
-    └─────────────────────────────────────────────────────────────┘
-
-    If installation fails, Binary stays in QUEUED with retry_at bumped.
-    """
-
-    model_attr_name = "binary"
-    binary: Binary
-
-    # States
-    queued = State(value=Binary.StatusChoices.QUEUED, initial=True)
-    installed = State(value=Binary.StatusChoices.INSTALLED, final=True)
-
-    # Tick Event - install happens during transition
-    tick = queued.to.itself(unless="can_install") | queued.to(installed, cond="can_install", on="on_install")
-
-    def can_install(self) -> bool:
-        """Check if binary installation can start."""
-        return bool(self.binary.name and self.binary.binproviders)
-
-    @queued.enter
-    def enter_queued(self):
-        """Binary is queued for installation."""
-        self.binary.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Binary.StatusChoices.QUEUED,
-        )
-
-    def on_install(self):
-        """Called during queued→installed transition. Runs installation synchronously."""
-        import sys
-
-        rprint(f"[cyan]      🔄 BinaryMachine.on_install() - installing {self.binary.name}[/cyan]", file=sys.stderr)
-
-        # Run installation hooks (synchronous, updates abspath/version/sha256 and sets status)
-        self.binary.run()
-
-        # Check if installation succeeded by looking at updated status
-        # Note: Binary.run() updates self.binary.status internally but doesn't refresh our reference
-        self.binary.refresh_from_db()
-
-        if self.binary.status != Binary.StatusChoices.INSTALLED:
-            # Installation failed - abort transition, stay in queued
-            rprint(f"[red]      ❌ BinaryMachine - {self.binary.name} installation failed, retrying later[/red]", file=sys.stderr)
-
-            # Bump retry_at to try again later
-            self.binary.update_and_requeue(
-                retry_at=timezone.now() + timedelta(seconds=300),  # Retry in 5 minutes
-                status=Binary.StatusChoices.QUEUED,  # Ensure we stay queued
-            )
-
-            # Increment health stats for failure
-            self.binary.increment_health_stats(success=False)
-
-            # Abort the transition - this will raise an exception and keep us in queued
-            raise Exception(f"Binary {self.binary.name} installation failed")
-
-        rprint(f"[cyan]      ✅ BinaryMachine - {self.binary.name} installed successfully[/cyan]", file=sys.stderr)
-
-    @installed.enter
-    def enter_installed(self):
-        """Binary installed successfully."""
-        self.binary.update_and_requeue(
-            retry_at=None,
-            status=Binary.StatusChoices.INSTALLED,
-        )
-
-        # Increment health stats
-        self.binary.increment_health_stats(success=True)
-
-
-# =============================================================================
-# Process State Machine
-# =============================================================================
-
-
-class ProcessMachine(BaseStateMachine):
-    """
-    State machine for managing Process (OS subprocess) lifecycle.
-
-    Process Lifecycle:
-    ┌─────────────────────────────────────────────────────────────┐
-    │ QUEUED State                                                │
-    │  • Process ready to launch, waiting for resources           │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() when can_start()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ RUNNING State → enter_running()                             │
-    │  1. process.launch()                                        │
-    │     • Spawn subprocess with cmd, pwd, env, timeout          │
-    │     • Set pid, started_at                                   │
-    │     • Process runs in background or foreground              │
-    │  2. Monitor process completion                              │
-    │     • Check exit code when process completes                │
-    └─────────────────────────────────────────────────────────────┘
-                            ↓ tick() checks is_exited()
-    ┌─────────────────────────────────────────────────────────────┐
-    │ EXITED State                                                │
-    │  • Process completed (exit_code set)                        │
-    │  • Health stats incremented                                 │
-    │  • stdout/stderr captured                                   │
-    └─────────────────────────────────────────────────────────────┘
-
-    Note: This is a simpler state machine than ArchiveResult.
-    Process is just about execution lifecycle. ArchiveResult handles
-    the archival-specific logic (status, output parsing, etc.).
-    """
-
-    model_attr_name = "process"
-    process: Process
-
-    # States
-    queued = State(value=Process.StatusChoices.QUEUED, initial=True)
-    running = State(value=Process.StatusChoices.RUNNING)
-    exited = State(value=Process.StatusChoices.EXITED, final=True)
-
-    # Tick Event - transitions based on conditions
-    tick = (
-        queued.to.itself(unless="can_start")
-        | queued.to(running, cond="can_start")
-        | running.to.itself(unless="is_exited")
-        | running.to(exited, cond="is_exited")
-    )
-
-    # Additional events (for explicit control)
-    launch = queued.to(running)
-    kill = running.to(exited)
-
-    def can_start(self) -> bool:
-        """Check if process can start (has cmd and machine)."""
-        return bool(self.process.cmd and self.process.machine)
-
-    def is_exited(self) -> bool:
-        """Check if process has exited (exit_code is set)."""
-        return self.process.exit_code is not None
-
-    @queued.enter
-    def enter_queued(self):
-        """Process is queued for execution."""
-        self.process.update_and_requeue(
-            retry_at=timezone.now(),
-            status=Process.StatusChoices.QUEUED,
-        )
-
-    @running.enter
-    def enter_running(self):
-        """Start process execution."""
-        # Lock the process while it runs
-        self.process.update_and_requeue(
-            retry_at=timezone.now() + timedelta(seconds=self.process.timeout),
-            status=Process.StatusChoices.RUNNING,
-            started_at=timezone.now(),
-        )
-
-        # Launch the subprocess
-        # NOTE: This is a placeholder - actual launch logic would
-        # be implemented based on how hooks currently spawn processes
-        # For now, Process is a data model that tracks execution metadata
-        # The actual subprocess spawning is still handled by run_hook()
-
-        # Mark as immediately exited for now (until we refactor run_hook)
-        # In the future, this would actually spawn the subprocess
-        self.process.exit_code = 0  # Placeholder
-        self.process.save()
-
-    @exited.enter
-    def enter_exited(self):
-        """Process has exited."""
-        self.process.update_and_requeue(
-            retry_at=None,
-            status=Process.StatusChoices.EXITED,
-            ended_at=timezone.now(),
-        )
-
-
-# =============================================================================
-# State Machine Registration
-# =============================================================================
-
-# Manually register state machines with python-statemachine registry
-registry.register(BinaryMachine)
-registry.register(ProcessMachine)

archivebox/misc/serve_static.py

@@ -33,6 +33,26 @@ from archivebox.misc.logging_util import printable_filesize
 _HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
 IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
 TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
+html {
+    width: 100%;
+    min-width: 100%;
+    background: #fff;
+}
+body {
+    box-sizing: border-box;
+    width: min(100%, 72rem);
+    max-width: none;
+    min-height: 100vh;
+    margin: 0 auto;
+    padding: clamp(1rem, 3vw, 2rem);
+    background: #fff;
+    color: #111827;
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+    line-height: 1.55;
+}
+body > * {
+    max-width: 100%;
+}
 img:not([width]):not([height]) {
     max-width: min(100%, 12rem);
     max-height: 12rem;

archivebox/plugins/discovery.py

@@ -1,12 +1,12 @@
 __package__ = "archivebox.plugins"
 
-import json
 from collections.abc import Iterable
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Protocol, TypedDict
 
 from abx_plugins import get_plugins_dir
+from abx_dl.catalog import PluginCatalog, PluginConfigResolver
 from django.utils.safestring import mark_safe
 
 from archivebox.config.constants import CONSTANTS
@@ -29,18 +29,18 @@ USER_PLUGINS_DIR = CONSTANTS.USER_PLUGINS_DIR
 
 
 def iter_plugin_dirs() -> list[Path]:
-    """Iterate over all built-in and user plugin directories."""
-    plugin_dirs: list[Path] = []
+    """Return the exact plugin directories exposed by the shared catalog."""
+    return [plugin.path for plugin in get_plugin_catalog().values()]
 
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
 
-        for plugin_dir in base_dir.iterdir():
-            if plugin_dir.is_dir() and not plugin_dir.name.startswith("_"):
-                plugin_dirs.append(plugin_dir)
+@lru_cache(maxsize=1)
+def get_plugin_catalog() -> PluginCatalog:
+    return PluginCatalog.discover(extra_plugin_dirs=[USER_PLUGINS_DIR], runtime="archivebox")
 
-    return plugin_dirs
+
+@lru_cache(maxsize=1)
+def get_plugin_config_resolver() -> PluginConfigResolver:
+    return PluginConfigResolver(get_plugin_catalog())
 
 
 @lru_cache(maxsize=1)
@@ -52,25 +52,11 @@ def get_plugins() -> list[str]:
     or a standardized templates/icon.html asset. This includes non-extractor
     plugins such as binary providers and shared base plugins.
     """
-    plugins = []
-
-    for plugin_dir in iter_plugin_dirs():
-        has_hooks = any(plugin_dir.glob("on_*__*.*"))
-        has_config = (plugin_dir / "config.json").exists()
-        has_icon = (plugin_dir / "templates" / "icon.html").exists()
-        if has_hooks or has_config or has_icon:
-            plugins.append(plugin_dir.name)
-
-    return sorted(set(plugins))
+    return sorted(get_plugin_catalog())
 
 
 def get_plugin_models():
-    from abx_dl.models import discover_plugins
-
-    plugins = {}
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        plugins.update(discover_plugins(plugins_dir=base_dir, runtime="archivebox"))
-    return plugins
+    return get_plugin_catalog().plugins
 
 
 def get_plugin_name(plugin: str) -> str:
@@ -99,91 +85,17 @@ def get_enabled_plugins(config: ConfigLookup | None = None, **config_kwargs: Any
 
         config = get_config(**config_kwargs)
 
-    enabled = []
-    disabled = []
-    for plugin in get_plugins():
-        plugin_config = get_plugin_special_config(plugin, config)
-        if plugin_config["enabled"]:
-            enabled.append(plugin)
-        else:
-            disabled.append(plugin)
-
-    from abx_dl.models import filter_plugins
-
-    return list(filter_plugins(get_plugin_models(), enabled, include_providers=True, disabled_names=disabled))
+    return get_plugin_config_resolver().enabled_plugin_names_from_flat(dict(config.items()))
 
 
-def discover_plugins_that_provide_interface(
-    module_name: str,
-    required_attrs: list[str],
-    plugin_prefix: str | None = None,
-) -> dict[str, Any]:
-    """
-    Discover plugins that provide a specific Python module with required interface.
-
-    This enables dynamic plugin discovery for features like search backends,
-    storage backends, etc. without hardcoding imports.
-    """
-    import importlib.util
-
-    backends = {}
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        for plugin_dir in base_dir.iterdir():
-            if not plugin_dir.is_dir():
-                continue
-
-            plugin_name = plugin_dir.name
-            if plugin_prefix and not plugin_name.startswith(plugin_prefix):
-                continue
-
-            module_path = plugin_dir / f"{module_name}.py"
-            if not module_path.exists():
-                continue
-
-            try:
-                spec = importlib.util.spec_from_file_location(
-                    f"archivebox.dynamic_plugins.{plugin_name}.{module_name}",
-                    module_path,
-                )
-                if spec is None or spec.loader is None:
-                    continue
-
-                module = importlib.util.module_from_spec(spec)
-                spec.loader.exec_module(module)
-
-                if not all(attr in vars(module) for attr in required_attrs):
-                    continue
-
-                if plugin_prefix:
-                    backend_name = plugin_name[len(plugin_prefix) :]
-                else:
-                    backend_name = plugin_name
-
-                backends[backend_name] = module
-
-            except Exception:
-                continue
-
-    return backends
-
-
-def get_search_backends() -> dict[str, Any]:
-    """
-    Discover all available search backend plugins.
-
-    Search backends must provide a search.py module with:
-        - search(query: str) -> List[str]  (returns snapshot IDs)
-        - flush(snapshot_ids: Iterable[str]) -> None
-    """
-    return discover_plugins_that_provide_interface(
-        module_name="search",
-        required_attrs=["search", "flush"],
-        plugin_prefix="search_backend_",
-    )
+def get_search_backends():
+    """Return plugins that declare both standalone search commands."""
+    catalog = get_plugin_catalog()
+    return {
+        plugin.name.removeprefix("search_backend_"): plugin
+        for plugin in catalog.values()
+        if catalog.command(plugin.name, "search") is not None and catalog.command(plugin.name, "flush") is not None
+    }
 
 
 @lru_cache(maxsize=1)
@@ -196,33 +108,7 @@ def discover_plugin_configs() -> dict[str, dict[str, Any]]:
     schemas are plugin package metadata, not live user config; runtime values
     still come from env/db config at each callsite.
     """
-    configs = {}
-
-    for plugin_dir in iter_plugin_dirs():
-        config_path = plugin_dir / "config.json"
-        if not config_path.exists():
-            continue
-
-        try:
-            with open(config_path) as f:
-                schema = json.load(f)
-
-            if not isinstance(schema, dict):
-                continue
-            if schema.get("type") != "object":
-                continue
-            if "properties" not in schema:
-                continue
-
-            configs[plugin_dir.name] = schema
-
-        except (json.JSONDecodeError, OSError) as e:
-            import sys
-
-            print(f"Warning: Failed to load config.json from {plugin_dir.name}: {e}", file=sys.stderr)
-            continue
-
-    return configs
+    return get_plugin_config_resolver().schemas
 
 
 def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited: set[str] | None = None) -> PluginSpecialConfig:
@@ -234,26 +120,7 @@ def get_plugin_special_config(plugin_name: str, config: ConfigLookup, _visited:
         - {PLUGIN}_TIMEOUT: Plugin-specific timeout (fallback to TIMEOUT, default 300)
         - {PLUGIN}_BINARY: Primary binary path (default to plugin_name)
     """
-    plugin_upper = plugin_name.upper()
-
-    enabled_key = f"{plugin_upper}_ENABLED"
-    enabled = config.get(enabled_key)
-    if enabled is None:
-        enabled = True
-    elif isinstance(enabled, str):
-        enabled = enabled.lower() not in ("false", "0", "no", "")
-
-    timeout_key = f"{plugin_upper}_TIMEOUT"
-    timeout = config.get(timeout_key) or config.get("TIMEOUT", 300)
-
-    binary_key = f"{plugin_upper}_BINARY"
-    binary = config.get(binary_key, plugin_name)
-
-    return {
-        "enabled": bool(enabled),
-        "timeout": int(timeout),
-        "binary": str(binary),
-    }
+    return get_plugin_config_resolver().runtime_settings(plugin_name, dict(config.items()))
 
 
 DEFAULT_TEMPLATES = {
@@ -295,11 +162,11 @@ def get_plugin_template(plugin: str, template_name: str, fallback: bool = True)
     if base_name in ("yt-dlp", "youtube-dl"):
         base_name = "ytdlp"
 
-    for plugin_dir in iter_plugin_dirs():
-        if plugin_dir.name == base_name or plugin_dir.name.endswith(f"_{base_name}"):
-            template_path = plugin_dir / "templates" / f"{template_name}.html"
-            if template_path.exists():
-                return template_path.read_text()
+    catalog = get_plugin_catalog()
+    if base_name in catalog:
+        template_path = catalog.template_path(base_name, template_name)
+        if template_path is not None:
+            return template_path.read_text()
 
     if fallback:
         return DEFAULT_TEMPLATES.get(template_name, "")

archivebox/plugins/forms.py

@@ -11,133 +11,19 @@ from django.utils.html import format_html
 
 from archivebox.config import CONSTANTS_CONFIG
 from archivebox.config.common import ArchiveBoxConfig, get_config
-from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
+from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_catalog, get_plugin_icon, get_plugins
 
 
 PLUGIN_CONFIG_FIELD_PREFIX = "plugin_config__"
-PLUGIN_GROUP_DEFINITIONS = (
-    (
-        "main_plugins",
-        "Main",
-        "",
-        "",
-        "",
-        (
-            "dom",
-            "screenshot",
-            "pdf",
-            "singlefile",
-            "wget",
-            "archivedotorg",
-            "chrome_mhtml",
-            "archivewebpage",
-        ),
-    ),
-    (
-        "page_setup_plugins",
-        "Page Setup",
-        "",
-        "",
-        "",
-        (
-            "chrome",
-            "infiniscroll",
-            "modalcloser",
-            "ublock",
-            "istilldontcareaboutcookies",
-            "twocaptcha",
-            "claudechrome",
-        ),
-    ),
-    (
-        "media_plugins",
-        "Media",
-        "",
-        "",
-        "",
-        (
-            "staticfile",
-            "responses",
-            "chrome_screencast",
-            "ytdlp",
-            "gallerydl",
-            "git",
-        ),
-    ),
-    (
-        "text_plugins",
-        "Text",
-        "",
-        "",
-        "",
-        (
-            "readability",
-            "htmltotext",
-            "defuddle",
-            "forumdl",
-            "mercury",
-            "trafilatura",
-            "liteparse",
-            "opendataloader",
-            "papersdl",
-        ),
-    ),
-    (
-        "metadata_plugins",
-        "Metadata",
-        "",
-        "",
-        "",
-        (
-            "title",
-            "favicon",
-            "headers",
-            "redirects",
-            "accessibility",
-            "consolelog",
-            "sslcerts",
-            "dns",
-            "seo",
-            "hashes",
-        ),
-    ),
-    (
-        "postprocessing_plugins",
-        "Postprocessing",
-        "",
-        "",
-        "",
-        (
-            "parse_dom_outlinks",
-            "parse_html_urls",
-            "parse_jsonl_urls",
-            "parse_netscape_urls",
-            "parse_rss_urls",
-            "parse_txt_urls",
-            "claudecode",
-            "claudecodecleanup",
-            "claudecodeextract",
-        ),
-    ),
+PLUGIN_GROUPS = (
+    ("main", "main_plugins", "Main"),
+    ("page_setup", "page_setup_plugins", "Page Setup"),
+    ("media", "media_plugins", "Media"),
+    ("text", "text_plugins", "Text"),
+    ("metadata", "metadata_plugins", "Metadata"),
+    ("postprocessing", "postprocessing_plugins", "Postprocessing"),
+    ("other", "other_plugins", "Other"),
 )
-HIDDEN_PLUGIN_CONFIG_UI_PLUGINS = {
-    "apt",
-    "base",
-    "bash",
-    "brew",
-    "cargo",
-    "chromewebstore",
-    "env",
-    "media",
-    "npm",
-    "opencode",
-    "pip",
-    "puppeteer",
-    "search_backend_ripgrep",
-    "search_backend_sonic",
-    "search_backend_sqlite",
-    "ssl",
-}
 TIMEOUT_INPUT_PATTERN = r"(0|[1-9][0-9]*|[0-9]+(?:\.[0-9]+)?\s*(?:s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours))"
 
 
@@ -255,26 +141,21 @@ class PluginConfigFormMixin:
     allow_crawl_execution_config_fields = True
 
     def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
-        all_plugins = get_plugins()
+        catalog = get_plugin_catalog()
+        all_plugins = set(catalog)
         plugin_configs = discover_plugin_configs()
         runtime_config = runtime_config or get_config()
         self.plugin_config_binary_urls = get_plugin_config_binary_urls(runtime_config)
-        grouped_plugins = set().union(*(group[-1] for group in PLUGIN_GROUP_DEFINITIONS))
-        other_plugins = tuple(sorted(set(all_plugins) - grouped_plugins - HIDDEN_PLUGIN_CONFIG_UI_PLUGINS))
+        grouped_plugins = catalog.groups()
 
-        for field_name, *_rest, plugin_names in PLUGIN_GROUP_DEFINITIONS:
+        group_specs = []
+        for category, field_name, title in PLUGIN_GROUPS:
+            plugin_names = tuple(plugin.name for plugin in grouped_plugins.get(category, []))
+            group_specs.append((field_name, title, "", "", "", plugin_names))
             if field_name in self.fields:
                 get_choice_field(self, field_name).choices = [
                     (p, get_plugin_choice_label(p, plugin_configs)) for p in plugin_names if p in all_plugins
                 ]
-
-        if "other_plugins" in self.fields:
-            get_choice_field(self, "other_plugins").choices = [(p, get_plugin_choice_label(p, plugin_configs)) for p in other_plugins]
-
-        group_specs = (
-            *PLUGIN_GROUP_DEFINITIONS,
-            ("other_plugins", "Other", "", "", "", other_plugins),
-        )
         binary_url_lookup = _build_required_binary_url_lookup(plugin_configs, runtime_config)
         self.plugin_groups = [
             {

archivebox/plugins/hooks.py

@@ -1,146 +1,32 @@
+"""ArchiveBox adapters around the framework-free abx-dl plugin runtime.
+
+Discovery and execution are owned by abx-dl. ArchiveBox keeps only the small
+Django projection adapter and its application-specific URL-output reader.
 """
-Hook discovery and execution helpers for ArchiveBox plugins.
 
-ArchiveBox no longer drives plugin execution itself during normal crawls.
-`abx-dl` owns the live runtime and emits typed bus events; ArchiveBox mainly:
-
-- discovers hook files for inspection / docs / legacy direct execution helpers
-- executes individual hook scripts when explicitly requested
-- parses hook stdout JSONL records into ArchiveBox models when needed
-
-Hook-backed event families are discovered from filenames like:
-    on_CrawlSetup__*
-    on_Snapshot__*
-
-Internal bus event names are normalized to the corresponding
-`on_{EventFamily}__*` prefix by a simple string transform. If no scripts exist
-for that prefix, discovery returns `[]`.
-
-Directory structure:
-    abx_plugins/plugins/<plugin_name>/on_<Event>__<hook_name>.<ext>     (built-in package)
-    data/custom_plugins/<plugin_name>/on_<Event>__<hook_name>.<ext>     (user)
-
-Hook contract:
-    Input:  --url=<url> (and other --key=value args)
-    Output: JSONL records to stdout, files to $PWD
-    Exit:   0 = success, non-zero = failure
-
-Execution order:
-    - Hooks are named with two-digit prefixes (00-99) and sorted lexicographically by filename
-    - Foreground hooks run sequentially in that order
-    - Background hooks (.bg suffix) run concurrently and do not block foreground progress
-    - After all foreground hooks complete, background hooks receive SIGTERM and must finalize
-
-Hook naming convention:
-    on_{EventFamily}__{run_order}_{description}[.bg].{ext}
-
-API:
-    discover_hooks(event)     -> List[Path]     Find hook scripts for a hook-backed event family
-    run_hook(script, ...)     -> Process        Execute a hook script directly
-    is_background_hook(name)  -> bool           Check if hook is background (.bg suffix)
-"""
+from __future__ import annotations
 
 __package__ = "archivebox.plugins"
 
-import json
-import os
-from collections.abc import Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, runtime_checkable
+from typing import Any
+
+from abx_dl.models import parse_hook_filename
 
-from archivebox.config.constants import CONSTANTS
-from archivebox.config.version import VERSION
 from archivebox.misc.util import fix_url_from_markdown, sanitize_extracted_url
-from archivebox.plugins.discovery import (
-    BUILTIN_PLUGINS_DIR,
-    USER_PLUGINS_DIR,
-    ConfigLookup,
-    get_enabled_plugins,
-    get_plugin_special_config,
-)
-
-if TYPE_CHECKING:
-    from archivebox.machine.models import Process
-
-
-@runtime_checkable
-class ConfigDump(Protocol):
-    def as_dict(self) -> dict[str, Any]: ...
-
-
-def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
-    return isinstance(config, ConfigDump)
-
-
-def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
-    if config is None:
-        return {}
-    if _has_config_dump(config):
-        return dict(config.as_dict())
-    return dict(config.items())
-
-
-# =============================================================================
-# Hook Step Extraction
-# =============================================================================
+from archivebox.plugins.discovery import ConfigLookup, get_enabled_plugins, get_plugin_catalog
 
 
 def is_background_hook(hook_name: str) -> bool:
-    """
-    Check if a hook is a background hook (doesn't block foreground progression).
-
-    Background hooks have '.bg.' in their filename before the extension.
-
-    Args:
-        hook_name: Hook filename (e.g., 'on_Snapshot__10_chrome_tab.daemon.bg.js')
-
-    Returns:
-        True if background hook, False if foreground.
-
-    Examples:
-        is_background_hook('on_Snapshot__10_chrome_tab.daemon.bg.js') -> True
-        is_background_hook('on_Snapshot__50_wget.py') -> False
-        is_background_hook('on_Snapshot__63_media.finite.bg.py') -> True
-    """
-    return ".bg." in hook_name or "__background" in hook_name
+    parsed = parse_hook_filename(Path(hook_name).name)
+    return bool(parsed and parsed[2])
 
 
 def normalize_hook_event_name(event_name: str) -> str | None:
-    """
-    Normalize a hook event family or event class name to its on_* prefix.
-
-    Examples:
-        CrawlSetupEvent -> CrawlSetup
-        SnapshotEvent -> Snapshot
-        BinaryEvent -> Binary
-        CrawlCleanupEvent -> CrawlCleanup
-    """
     normalized = str(event_name or "").strip()
     if not normalized:
         return None
-
-    if normalized.endswith("Event"):
-        return normalized[:-5] or None
-    return normalized
-
-
-def _model_output_dir_from_child_path(path: Path, marker: str) -> Path | None:
-    """
-    Infer the model output dir from a model dir or one of its plugin subdirs.
-
-    Current ArchiveBox snapshot/crawl dirs are:
-        .../{snapshots,crawls}/YYYYMMDD/domain/uuid[/plugin]
-    """
-    parts = path.resolve().parts
-    try:
-        marker_index = parts.index(marker)
-    except ValueError:
-        return None
-
-    model_end_index = marker_index + 4
-    if len(parts) < model_end_index:
-        return None
-    return Path(*parts[:model_end_index])
+    return normalized.removesuffix("Event") or None
 
 
 def discover_hooks(
@@ -149,481 +35,40 @@ def discover_hooks(
     config: ConfigLookup | None = None,
     **config_kwargs: Any,
 ) -> list[Path]:
-    """
-    Find all hook scripts for an event family.
-
-    Searches both built-in and user plugin directories.
-    Filters out hooks from disabled plugins by default (respects USE_/SAVE_ flags).
-    Returns scripts sorted alphabetically by filename for deterministic execution order.
-
-    Hook naming convention uses numeric prefixes to control order:
-        on_Snapshot__10_title.py        # runs first
-        on_Snapshot__15_singlefile.py   # runs second
-        on_Snapshot__26_readability.py  # runs later (depends on singlefile)
-
-    Args:
-        event_name: Hook event family or event class name.
-            Examples: 'CrawlSetupEvent', 'Snapshot'.
-            Event names are normalized by stripping a trailing `Event`.
-            If no matching `on_{EventFamily}__*` scripts exist, returns [].
-        filter_disabled: If True, skip hooks from disabled plugins (default: True)
-        config: Optional pre-merged config dict from get_config().
-        **config_kwargs: Scope/override args forwarded to get_config() when config is not supplied.
-
-    Returns:
-        Sorted list of hook script paths from enabled plugins only.
-
-    Examples:
-        # With proper config context (recommended):
-        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot)
-        discover_hooks('Snapshot', config=config)
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ...] (wget excluded if SAVE_WGET=False)
-
-        # Without config (uses global defaults):
-        discover_hooks('Snapshot')
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ...]
-
-        # Show all plugins regardless of enabled status:
-        discover_hooks('Snapshot', filter_disabled=False)
-        # Returns: [Path('.../on_Snapshot__10_title.py'), ..., Path('.../on_Snapshot__50_wget.py')]
-    """
-    hook_event_name = normalize_hook_event_name(event_name)
-    if not hook_event_name:
+    """Return the exact catalog hooks used by abx-dl, in execution order."""
+    normalized = normalize_hook_event_name(event_name)
+    if not normalized or normalized == "BinaryRequest":
         return []
-    if hook_event_name == "BinaryRequest":
-        return []
-
-    hooks = []
-
-    for base_dir in (BUILTIN_PLUGINS_DIR, USER_PLUGINS_DIR):
-        if not base_dir.exists():
-            continue
-
-        # Search for hook scripts in all subdirectories
-        for ext in ("sh", "py", "js"):
-            pattern = f"*/on_{hook_event_name}__*.{ext}"
-            hooks.extend(base_dir.glob(pattern))
-
-            # Also check for hooks directly in the plugins directory
-            pattern_direct = f"on_{hook_event_name}__*.{ext}"
-            hooks.extend(base_dir.glob(pattern_direct))
-
+    names = None
     if filter_disabled:
-        # Get merged config if not provided (lazy import to avoid circular dependency)
         if config is None:
             from archivebox.config.common import get_config
 
             config = get_config(**config_kwargs)
-
-        enabled_plugins = set(get_enabled_plugins(config=config))
-        enabled_hooks = []
-
-        for hook in hooks:
-            # Get plugin name from parent directory
-            # e.g., abx_plugins/plugins/wget/on_Snapshot__50_wget.py -> 'wget'
-            plugin_name = hook.parent.name
-
-            # Check if this is a plugin directory (not the root plugins dir)
-            if hook.parent.resolve() in (BUILTIN_PLUGINS_DIR.resolve(), USER_PLUGINS_DIR.resolve()):
-                # Hook is in root plugins directory, not a plugin subdir
-                # Include it by default (no filtering for non-plugin hooks)
-                enabled_hooks.append(hook)
-                continue
-
-            if plugin_name in enabled_plugins:
-                enabled_hooks.append(hook)
-
-        hooks = enabled_hooks
-
-    # Sort by filename (not full path) to ensure numeric prefix ordering works
-    # e.g., on_Snapshot__10_title.py sorts before on_Snapshot__26_readability.py
-    return sorted(set(hooks), key=lambda p: p.name)
-
-
-def run_hook(
-    script: Path,
-    output_dir: Path,
-    config: ConfigLookup | Mapping[str, Any] | None = None,
-    timeout: int | None = None,
-    parent: Optional["Process"] = None,
-    **kwargs: Any,
-) -> "Process":
-    """
-    Execute a hook script with the given arguments using Process model.
-
-    This is the low-level hook executor that creates a Process record and
-    uses Process.launch() for subprocess management.
-
-    Config is passed to hooks via environment variables. Crawl/snapshot callers
-    should pass the runtime config produced by for_crawl_runtime().
-
-    Args:
-        script: Path to the hook script (.sh, .py, or .js)
-        output_dir: Working directory for the script (where output files go)
-        config: Optional runtime config dict from for_crawl_runtime().
-                If omitted, pass scope/override args using kwargs prefixed with config_.
-        timeout: Maximum execution time in seconds
-                 If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
-        parent: Optional parent Process (for tracking worker->hook hierarchy)
-        **kwargs: Arguments passed to the script as --key=value
-
-    Returns:
-        Process model instance (use process.exit_code, process.stdout, process.get_records())
-
-    Example:
-        from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot).for_crawl_runtime(crawl=my_crawl, snapshot=my_snapshot)
-        process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
-        if process.status == 'exited':
-            records = process.get_records()  # Get parsed JSONL output
-    """
-    from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import (
-        ArchiveBoxConfig,
-        _archivebox_config_input_names,
-        get_config,
-        normalize_runtime_config,
-        _plugin_enabled_config_keys,
-    )
-
-    config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
-    config_overrides = _config_to_overrides(config)
-    explicit_override_keys = set(config_overrides)
-    resolved_config = get_config(overrides=config_overrides, **config_scope)
-    hook_config = normalize_runtime_config(
-        resolved_config.for_crawl_runtime(runtime_overrides=config_overrides),
-        json_safe=False,
-    )
-    hook_config.update(normalize_runtime_config(config_overrides, json_safe=False))
-    plugin_enabled_keys = set(_plugin_enabled_config_keys().values())
-    if plugin_enabled_keys.intersection(hook_config):
-        for enabled_key in plugin_enabled_keys:
-            hook_config.setdefault(enabled_key, False)
-
-    # Auto-detect timeout from plugin config if not explicitly provided
-    if timeout is None:
-        plugin_name = script.parent.name
-        plugin_config = get_plugin_special_config(plugin_name, resolved_config)
-        timeout = plugin_config["timeout"]
-    if timeout:
-        timeout = min(int(timeout), int(CONSTANTS.MAX_HOOK_RUNTIME_SECONDS))
-
-    # Get current machine
-    machine = Machine.current()
-    iface = NetworkInterface.current(refresh=True)
-    machine = iface.machine
-
-    # Auto-detect parent process if not explicitly provided
-    # This enables automatic hierarchy tracking: Worker -> Hook
-    if parent is None:
-        try:
-            parent = Process.current()
-        except Exception:
-            # If Process.current() fails (e.g., not in a worker context), leave parent as None
-            pass
-
-    if not script.is_file():
-        raise FileNotFoundError(f"Hook script not found: {script}")
-
-    # Hooks are opaque executables. Their shipped abxpkg shebang owns runtime
-    # and dependency resolution just as it does under abx-dl.
-    cmd = [str(script)]
-
-    # Build CLI arguments from kwargs
-    for key, value in kwargs.items():
-        # Skip keys that start with underscore (internal parameters)
-        if key.startswith("_"):
-            continue
-
-        arg_key = f"--{key.replace('_', '-')}"
-        if isinstance(value, bool):
-            if value:
-                cmd.append(arg_key)
-        elif value is not None and value != "":
-            # JSON-encode complex values, use str for simple ones
-            # Skip empty strings to avoid --key= which breaks argument parsers
-            if isinstance(value, (dict, list)):
-                cmd.append(f"{arg_key}={json.dumps(value)}")
-            else:
-                # Ensure value is converted to string and strip whitespace
-                str_value = str(value).strip()
-                if str_value:  # Only add if non-empty after stripping
-                    cmd.append(f"{arg_key}={str_value}")
-
-    # Set up environment with base paths
-    env = os.environ.copy()
-    archivebox_config_input_names = _archivebox_config_input_names()
-    for key in archivebox_config_input_names:
-        env.pop(key, None)
-    env.pop("PLUGINS", None)
-    env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
-    env["LIBRARY_VERSION"] = VERSION
-    env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
-    snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
-    crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
-    if snap_dir:
-        env["SNAP_DIR"] = str(snap_dir)
-    if crawl_dir:
-        env["CRAWL_DIR"] = str(crawl_dir)
-
-    # Export the runtime library root; abx-dl/abxpkg own executable lookup env.
-    lib_dir = hook_config.get("ABXPKG_LIB_DIR")
-    if lib_dir:
-        env["ABXPKG_LIB_DIR"] = str(lib_dir)
-
-    # Set Node.js module resolution paths.
-    # NODE_PATH may be a path list, but NODE_MODULES_DIR is a single canonical directory.
-    node_modules_dir = hook_config.get("NODE_MODULES_DIR")
-    if lib_dir and "ABXPKG_LIB_DIR" in explicit_override_keys and "NODE_MODULES_DIR" not in explicit_override_keys:
-        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
-    elif not node_modules_dir and lib_dir:
-        node_modules_dir = Path(lib_dir) / "pnpm" / "packages" / "chrome" / "node_modules"
-
-    node_path_parts = [part for part in str(hook_config.get("NODE_PATH") or "").split(os.pathsep) if part]
-    if node_modules_dir:
-        node_modules_dir = Path(node_modules_dir)
-        node_modules_dir.mkdir(parents=True, exist_ok=True)
-        node_modules_dir_str = str(node_modules_dir)
-        env["NODE_MODULES_DIR"] = node_modules_dir_str
-        env["NODE_MODULE_DIR"] = node_modules_dir_str
-        if node_modules_dir_str not in node_path_parts:
-            node_path_parts.append(node_modules_dir_str)
-    if node_path_parts:
-        env["NODE_PATH"] = os.pathsep.join(node_path_parts)
-
-    # Export all config values to environment (already merged by get_config())
-    # Skip keys we've already handled specially above (PATH, ABXPKG_LIB_DIR, NODE_PATH, etc.)
-    SKIP_KEYS = {
-        "PATH",
-        "ABXPKG_LIB_DIR",
-        "NODE_PATH",
-        "NODE_MODULES_DIR",
-        "NODE_MODULE_DIR",
-        "DATA_DIR",
-        "MACHINE_ID",
-        "SNAP_DIR",
-        "CRAWL_DIR",
-    }
-    canonical_config_keys = set(ArchiveBoxConfig.model_fields)
-    for key, value in hook_config.items():
-        if key in SKIP_KEYS:
-            continue  # Already handled specially above, don't overwrite
-        if key in archivebox_config_input_names and key not in canonical_config_keys:
-            continue
-        if value is None:
-            continue
-        elif isinstance(value, bool):
-            env[key] = "true" if value else "false"
-        elif isinstance(value, (list, dict)):
-            env[key] = json.dumps(value)
-        else:
-            env[key] = str(value)
-
-    # Create output directory if needed
-    output_dir.mkdir(parents=True, exist_ok=True)
-
-    # Detect if this is a background hook.
-    # Background hooks use the .bg. filename marker.
-    # Old convention: __background in stem (for backwards compatibility)
-    is_background = ".bg." in script.name or "__background" in script.stem
-
-    try:
-        # Create Process record
-        process = Process.objects.create(
-            machine=machine,
-            iface=iface,
-            parent=parent,
-            process_type=Process.TypeChoices.HOOK,
-            pwd=str(output_dir),
-            cmd=cmd,
-            timeout=timeout,
-        )
-
-        # Copy the env dict we already built (includes os.environ + all customizations)
-        process.env = env.copy()
-        process.hydrate_binary_from_context(plugin_name=script.parent.name, hook_path=str(script))
-
-        # Save env before launching
-        process.save()
-
-        # Launch subprocess using Process.launch()
-        process.launch(background=is_background)
-
-        # Return Process object (caller can use process.exit_code, process.stdout, process.get_records())
-        return process
-
-    except Exception as e:
-        # Create a failed Process record for exceptions
-        process = Process.objects.create(
-            machine=machine,
-            iface=iface,
-            process_type=Process.TypeChoices.HOOK,
-            pwd=str(output_dir),
-            cmd=cmd,
-            timeout=timeout,
-            status=Process.StatusChoices.EXITED,
-            exit_code=1,
-            stderr=f"Failed to run hook: {type(e).__name__}: {e}",
-        )
-        return process
-
-
-def extract_records_from_process(process: "Process") -> list[dict[str, Any]]:
-    """
-    Extract JSONL records from a Process's stdout.
-
-    Adds plugin metadata to each record.
-
-    Args:
-        process: Process model instance with stdout captured
-
-    Returns:
-        List of parsed JSONL records with plugin metadata
-    """
-    records = process.get_records()
-    if not records:
-        return []
-
-    # Extract plugin metadata from process.pwd and the shipped hook path in cmd.
-    # Python hooks execute directly through their shebang, while JS and shell
-    # hooks have an interpreter in cmd[0].
-    plugin_name = Path(process.pwd).name if process.pwd else "unknown"
-    plugin_hook = next((str(arg) for arg in process.cmd if Path(str(arg)).name.startswith("on_")), "")
-    hook_name = Path(plugin_hook).name if plugin_hook else "unknown"
-
-    for record in records:
-        # Add plugin metadata to record
-        record.setdefault("plugin", plugin_name)
-        record.setdefault("hook_name", hook_name)
-        record.setdefault("plugin_hook", plugin_hook)
-
-    return records
+        names = get_enabled_plugins(config=config)
+    return [hook.path for _plugin, hook in get_plugin_catalog().hooks(normalized, names=names)]
 
 
 def collect_urls_from_plugins(snapshot_dir: Path) -> list[dict[str, Any]]:
-    """
-    Collect all urls.jsonl entries from parser plugin output subdirectories.
-
-    Each parser plugin outputs urls.jsonl to its own subdir:
-        snapshot_dir/parse_rss_urls/urls.jsonl
-        snapshot_dir/parse_html_urls/urls.jsonl
-        etc.
-
-    This is not special handling - urls.jsonl is just a normal output file.
-    This utility collects them all for the crawl system.
-    """
-    urls = []
-
-    # Look in each immediate subdirectory for urls.jsonl
+    """Read the durable urls.jsonl interface emitted by parser plugins."""
+    urls: list[dict[str, Any]] = []
     if not snapshot_dir.exists():
         return urls
 
+    from archivebox.machine.models import Process
+
     for subdir in snapshot_dir.iterdir():
-        if not subdir.is_dir():
-            continue
-
         urls_file = subdir / "urls.jsonl"
-        if not urls_file.exists():
+        if not subdir.is_dir() or not urls_file.is_file():
             continue
-
         try:
-            from archivebox.machine.models import Process
-
-            text = urls_file.read_text()
-            for entry in Process.parse_records_from_text(text):
-                if entry.get("url"):
-                    entry["url"] = sanitize_extracted_url(fix_url_from_markdown(str(entry["url"]).strip()))
-                    if not entry["url"]:
-                        continue
-                    # Track which parser plugin found this URL
+            for entry in Process.parse_records_from_text(urls_file.read_text()):
+                if not entry.get("url"):
+                    continue
+                entry["url"] = sanitize_extracted_url(fix_url_from_markdown(str(entry["url"]).strip()))
+                if entry["url"]:
                     entry["plugin"] = subdir.name
                     urls.append(entry)
-        except Exception:
-            pass
-
+        except (OSError, UnicodeError):
+            continue
     return urls
-
-
-# =============================================================================
-# Hook Result Processing Helpers
-# =============================================================================
-
-
-def process_hook_records(records: list[dict[str, Any]], overrides: dict[str, Any] | None = None) -> dict[str, int]:
-    """
-    Process JSONL records emitted by hook stdout.
-
-    This handles hook-emitted record types such as Snapshot, Tag, and Binary.
-    It does not process internal bus lifecycle events, since those
-    are not emitted as JSONL records by hook subprocesses.
-
-    Args:
-        records: List of JSONL record dicts from result['records']
-        overrides: Dict with 'snapshot', 'crawl', 'dependency', 'created_by_id', etc.
-
-    Returns:
-        Dict with counts by record type
-    """
-    stats = {}
-    overrides = overrides or {}
-
-    for record in records:
-        record_type = record.get("type")
-        if not record_type:
-            continue
-
-        # Skip ArchiveResult records (they update the calling ArchiveResult, not create new ones)
-        if record_type == "ArchiveResult":
-            continue
-
-        try:
-            # Dispatch to appropriate model's from_json() method
-            if record_type == "Snapshot":
-                from archivebox.core.models import Snapshot
-
-                if record.get("url"):
-                    record = {
-                        **record,
-                        "url": sanitize_extracted_url(fix_url_from_markdown(str(record["url"]).strip())),
-                    }
-                    if not record["url"]:
-                        continue
-
-                # Check if discovered snapshot exceeds crawl max_depth
-                snapshot_depth = record.get("depth", 0)
-                crawl = overrides.get("crawl")
-                if crawl and snapshot_depth > crawl.max_depth:
-                    # Skip - this URL was discovered but exceeds max crawl depth
-                    continue
-
-                obj = Snapshot.from_json(record.copy(), overrides)
-                if obj:
-                    stats["Snapshot"] = stats.get("Snapshot", 0) + 1
-
-            elif record_type == "Tag":
-                from archivebox.core.models import Tag
-
-                obj = Tag.from_json(record.copy(), overrides)
-                if obj:
-                    stats["Tag"] = stats.get("Tag", 0) + 1
-
-            elif record_type == "Binary":
-                from archivebox.machine.models import Binary
-
-                obj = Binary.from_json(record.copy(), overrides)
-                if obj:
-                    stats[record_type] = stats.get(record_type, 0) + 1
-
-            else:
-                import sys
-
-                print(f"Warning: Unknown record type '{record_type}' from hook output", file=sys.stderr)
-
-        except Exception as e:
-            import sys
-
-            print(f"Warning: Failed to create {record_type}: {e}", file=sys.stderr)
-            continue
-
-    return stats

archivebox/progressmonitor/views.py

@@ -30,12 +30,16 @@ def progress_endpoint(scope: Literal["crawl", "snapshot"] | None = None, object_
 @lru_cache(maxsize=1)
 def _live_progress_plugin_names() -> tuple[frozenset[str], frozenset[str]]:
     plugin_configs = discover_plugin_configs()
+    indexing_plugin_names = frozenset(
+        plugin_name
+        for plugin_name, plugin_config in plugin_configs.items()
+        if {"search", "flush"}.issubset(plugin_config.get("commands", {}))
+    )
     download_plugin_names = frozenset(
         plugin_name
         for plugin_name, plugin_config in plugin_configs.items()
-        if plugin_config.get("output_mimetypes") and not plugin_name.startswith("search_backend_")
+        if plugin_config.get("output_mimetypes") and plugin_name not in indexing_plugin_names
     )
-    indexing_plugin_names = frozenset(plugin_name for plugin_name in plugin_configs if plugin_name.startswith("search_backend_"))
     return download_plugin_names, indexing_plugin_names
 
 

archivebox/search/backends.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox.search"
 
+import json
 import os
-from contextlib import contextmanager
 from typing import Any
 
 from archivebox.config.common import get_config
@@ -10,29 +10,21 @@ from archivebox.config.common import get_config
 _search_backends_cache: dict | None = None
 
 
-@contextmanager
-def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
-    """Temporarily expose resolved search config through os.environ for backend code."""
+def search_backend_command_env(config: dict[str, Any] | None = None, **config_kwargs: Any) -> dict[str, str]:
+    """Serialize resolved application config for a standalone plugin command."""
     config = config or get_config(**config_kwargs)
-    updates = {}
+    env = os.environ.copy()
     for key, value in config.items():
         key = str(key)
-        if not (key.startswith("SEARCH_BACKEND_") or key.endswith("_BINARY")):
-            continue
         if value is None:
             continue
-        if isinstance(value, (str, int, float, bool, os.PathLike)):
-            updates[key] = str(value)
-    previous = {key: os.environ.get(key) for key in updates}
-    os.environ.update(updates)
-    try:
-        yield
-    finally:
-        for key, value in previous.items():
-            if value is None:
-                os.environ.pop(key, None)
-            else:
-                os.environ[key] = value
+        if isinstance(value, bool):
+            env[key] = "true" if value else "false"
+        elif isinstance(value, (dict, list, tuple)):
+            env[key] = json.dumps(value)
+        elif isinstance(value, (str, int, float, os.PathLike)):
+            env[key] = str(value)
+    return env
 
 
 def normalize_search_backend_name(backend_name: str | None) -> str:
@@ -41,7 +33,7 @@ def normalize_search_backend_name(backend_name: str | None) -> str:
 
 
 def get_available_backends() -> dict:
-    """Discover search backend plugin modules and cache them in memory."""
+    """Discover search-capable plugins and cache their catalog entries."""
     global _search_backends_cache
 
     if _search_backends_cache is None:
@@ -53,7 +45,7 @@ def get_available_backends() -> dict:
 
 
 def get_backend(config: dict[str, Any] | None = None, **config_kwargs: Any) -> Any:
-    """Resolve the configured search backend module."""
+    """Resolve the configured search-capable plugin."""
     config = config or get_config(**config_kwargs)
     backend_name = normalize_search_backend_name(config.SEARCH_BACKEND_ENGINE)
     backends = get_available_backends()

archivebox/search/query.py

@@ -8,7 +8,7 @@ from django.db.models import Case, IntegerField, Q, QuerySet, Value, When
 from archivebox.config.common import get_config
 from archivebox.misc.logging import stderr
 from archivebox.misc.util import enforce_types
-from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_env
+from archivebox.search.backends import get_available_backends, get_backend, normalize_search_backend_name, search_backend_command_env
 from archivebox.search.config import get_search_mode, get_search_mode_backend, get_search_mode_base
 
 
@@ -278,18 +278,27 @@ def iter_query_search_ids(
                     # --csv/--json. Keep daemon diagnostics on stderr.
                     with redirect_stdout(sys.stderr):
                         ensure_daemon_stack(reason="search query")
-                with search_backend_env(config=config):
-                    if backend_name == "ripgrep":
-                        ids = backend.iter_search(query, search_mode=search_mode_base)
-                    else:
-                        ids = backend.search(query)
-                    for snapshot_id in ids:
-                        if snapshot_id in seen:
-                            continue
-                        seen.add(snapshot_id)
-                        yield snapshot_id
-                        if max_results and len(seen) >= max_results:
-                            return
+                from abx_dl.execution import iter_plugin_command
+                from archivebox.config.constants import CONSTANTS
+                from archivebox.plugins.discovery import get_plugin_catalog
+
+                command = get_plugin_catalog().command(backend.name, "search")
+                if command is None:
+                    raise RuntimeError(f'Plugin "{backend.name}" does not expose a search command')
+                ids = iter_plugin_command(
+                    command,
+                    arguments={"query": query, "search_mode": search_mode_base},
+                    env=search_backend_command_env(config=config),
+                    cwd=CONSTANTS.DATA_DIR,
+                    timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+                )
+                for snapshot_id in ids:
+                    if snapshot_id in seen:
+                        continue
+                    seen.add(snapshot_id)
+                    yield snapshot_id
+                    if max_results and len(seen) >= max_results:
+                        return
                 successful_backends += 1
             except Exception as err:
                 errors.append(err)
@@ -318,8 +327,22 @@ def flush_search_index(snapshots: QuerySet, config: dict[str, Any] | None = None
     snapshot_pks = [str(pk) for pk in snapshots.values_list("pk", flat=True)]
 
     try:
-        with search_backend_env(config=config):
-            backend.flush(snapshot_pks)
+        from abx_dl.execution import iter_plugin_command
+        from archivebox.config.constants import CONSTANTS
+        from archivebox.plugins.discovery import get_plugin_catalog
+
+        command = get_plugin_catalog().command(backend.name, "flush")
+        if command is None:
+            raise RuntimeError(f'Plugin "{backend.name}" does not expose a flush command')
+        list(
+            iter_plugin_command(
+                command,
+                stdin=snapshot_pks,
+                env=search_backend_command_env(config=config),
+                cwd=CONSTANTS.DATA_DIR,
+                timeout=max(1, int(config.get("TIMEOUT", 60))) * 4,
+            ),
+        )
     except Exception as err:
         stderr()
         stderr(

archivebox/services/archive_result_service.py

@@ -4,23 +4,19 @@ import asyncio
 import inspect
 import json
 import os
-import re
 import signal
 import sys
 import time
-from collections import defaultdict
-from collections.abc import Iterable
 from contextlib import contextmanager
 from functools import wraps
 from pathlib import Path
-from typing import Any, Protocol, runtime_checkable
+from typing import Any
 
 from asgiref.sync import sync_to_async
-from django.db import IntegrityError
 from django.utils import timezone
 
 from abx_dl.events import PROCESS_EXIT_SKIPPED, ArchiveResultEvent, ProcessCompletedEvent, ProcessStartedEvent, SnapshotEvent
-from abx_dl.output_files import guess_mimetype
+from abx_dl.output_files import OutputManifest
 from abx_dl.services.base import BaseService
 
 from .process_service import parse_event_datetime
@@ -72,129 +68,15 @@ def _perf_span(label: str):
         print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
 
 
-@runtime_checkable
-class ModelDumpable(Protocol):
-    def model_dump(self) -> dict[str, Any]: ...
-
-
-def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
-    exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
-    output_files: dict[str, dict] = {}
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-
-    if not plugin_dir.exists():
-        return output_files, total_size, ""
-
-    for file_path in plugin_dir.rglob("*"):
-        if not file_path.is_file():
-            continue
-        if ".hooks" in file_path.parts:
-            continue
-        if file_path.name in exclude_names:
-            continue
-        try:
-            stat = file_path.stat()
-        except OSError:
-            continue
-        mime_type = guess_mimetype(file_path) or "application/octet-stream"
-        relative_path = str(file_path.relative_to(plugin_dir))
-        output_files[relative_path] = {
-            "extension": file_path.suffix.lower().lstrip("."),
-            "mimetype": mime_type,
-            "size": stat.st_size,
-        }
-        mime_sizes[mime_type] += stat.st_size
-        total_size += stat.st_size
-
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return output_files, total_size, output_mimetypes
-
-
-def _coerce_output_file_size(value: Any) -> int:
-    try:
-        return max(int(value or 0), 0)
-    except (TypeError, ValueError):
-        return 0
-
-
-def _normalize_output_files(raw_output_files: Any) -> dict[str, dict]:
-    def _enrich_metadata(path: str, metadata: dict[str, Any]) -> dict[str, Any]:
-        normalized = dict(metadata)
-        if "extension" not in normalized:
-            normalized["extension"] = Path(path).suffix.lower().lstrip(".")
-        if "mimetype" not in normalized:
-            guessed = guess_mimetype(path)
-            if guessed:
-                normalized["mimetype"] = guessed
-        return normalized
-
-    if raw_output_files is None:
-        return {}
-
-    if isinstance(raw_output_files, str):
-        try:
-            raw_output_files = json.loads(raw_output_files)
-        except json.JSONDecodeError:
-            return {}
-
-    if isinstance(raw_output_files, dict):
-        normalized: dict[str, dict] = {}
-        for path, metadata in raw_output_files.items():
-            if not path:
-                continue
-            metadata_dict = dict(metadata) if isinstance(metadata, dict) else {}
-            metadata_dict.pop("path", None)
-            normalized[str(path)] = _enrich_metadata(str(path), metadata_dict)
-        return normalized
-
-    if not isinstance(raw_output_files, Iterable):
-        return {}
-
-    normalized: dict[str, dict] = {}
-    for item in raw_output_files:
-        if isinstance(item, str):
-            normalized[item] = _enrich_metadata(item, {})
-            continue
-        if isinstance(item, ModelDumpable):
-            item = item.model_dump()
-        if not isinstance(item, dict):
-            continue
-        path = str(item.get("path") or "").strip()
-        if not path:
-            continue
-        normalized[path] = _enrich_metadata(path, {key: value for key, value in item.items() if key != "path" and value not in (None, "")})
-
-    return normalized
-
-
-def _has_structured_output_metadata(output_files: dict[str, dict]) -> bool:
-    return any(any(key in metadata for key in ("extension", "mimetype", "size")) for metadata in output_files.values())
-
-
-def _summarize_output_files(output_files: dict[str, dict]) -> tuple[int, str]:
-    mime_sizes: dict[str, int] = defaultdict(int)
-    total_size = 0
-
-    for metadata in output_files.values():
-        if not isinstance(metadata, dict):
-            continue
-        size = _coerce_output_file_size(metadata.get("size"))
-        mimetype = str(metadata.get("mimetype") or "").strip()
-        total_size += size
-        if mimetype and size:
-            mime_sizes[mimetype] += size
-
-    output_mimetypes = ",".join(mime for mime, _size in sorted(mime_sizes.items(), key=lambda item: item[1], reverse=True))
-    return total_size, output_mimetypes
+def _manifest_metadata(manifest: OutputManifest) -> tuple[dict[str, dict], int, str]:
+    return manifest.as_mapping(), manifest.total_size, ",".join(manifest.mimetypes)
 
 
 def _resolve_output_metadata(raw_output_files: Any, plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
-    normalized_output_files = _normalize_output_files(raw_output_files)
-    if normalized_output_files and _has_structured_output_metadata(normalized_output_files):
-        output_size, output_mimetypes = _summarize_output_files(normalized_output_files)
-        return normalized_output_files, output_size, output_mimetypes
-    return _collect_output_metadata(plugin_dir)
+    manifest = OutputManifest.from_value(raw_output_files)
+    if manifest.files and any(output_file.size for output_file in manifest.files):
+        return _manifest_metadata(manifest)
+    return _manifest_metadata(OutputManifest.scan(plugin_dir, containment_root=plugin_dir.parent))
 
 
 def _normalize_status(status: str) -> str:
@@ -203,11 +85,6 @@ def _normalize_status(status: str) -> str:
     return status or "failed"
 
 
-def _snapshot_hook_order(hook_name: str) -> int:
-    match = re.match(r"^on_Snapshot__(\d+)", hook_name or "")
-    return int(match.group(1)) if match else -1
-
-
 def _normalize_snapshot_title(candidate: str, *, snapshot_url: str) -> str:
     title = " ".join(line.strip() for line in str(candidate or "").splitlines() if line.strip()).strip()
     if not title:
@@ -325,7 +202,6 @@ def _save_archiveresult_event_to_db(
         start_ts = parse_event_datetime(event.start_ts)
         end_ts = parse_event_datetime(event.end_ts) or timezone.now()
         defaults = {
-            "hook_name": event.hook_name,
             "status": _normalize_status(event.status),
             "output_str": event.output_str,
             "output_json": event.output_json,
@@ -340,47 +216,13 @@ def _save_archiveresult_event_to_db(
         if event.error:
             defaults["notes"] = event.error
 
-    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
-        result = ArchiveResult.objects.filter(
-            snapshot=snapshot,
-            plugin=event.plugin,
-        ).first()
-    if result is None:
-        try:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
-                result = ArchiveResult.objects.create(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                    **defaults,
-                )
-        except IntegrityError:
-            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
-                result = ArchiveResult.objects.get(
-                    snapshot=snapshot,
-                    plugin=event.plugin,
-                )
-
-    if result.output_files:
-        merged_output_files = {**result.output_files, **defaults["output_files"]}
-        defaults["output_files"] = merged_output_files
-        defaults["output_size"], defaults["output_mimetypes"] = _summarize_output_files(merged_output_files)
-        defaults["output_size"] = max(defaults["output_size"], int(result.output_size or 0))
-        defaults["output_mimetypes"] = ",".join(
-            dict.fromkeys(
-                mimetype.strip()
-                for value in (defaults["output_mimetypes"], result.output_mimetypes)
-                for mimetype in value.split(",")
-                if mimetype.strip()
-            ),
+    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_or_create"):
+        result, _created = ArchiveResult.get_or_create_by_hook(
+            snapshot,
+            event.plugin,
+            event.hook_name,
+            defaults=defaults,
         )
-    if result.start_ts and defaults["start_ts"]:
-        defaults["start_ts"] = min(result.start_ts, defaults["start_ts"])
-    if result.end_ts and defaults["end_ts"]:
-        defaults["end_ts"] = max(result.end_ts, defaults["end_ts"])
-    if _snapshot_hook_order(result.hook_name) > _snapshot_hook_order(event.hook_name):
-        for field in ("hook_name", "status", "output_str", "output_json", "process_id", "notes"):
-            if field in result.__dict__:
-                defaults[field] = result.__dict__[field]
 
     with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
         update_fields = []
@@ -422,7 +264,7 @@ def _save_archiveresult_event_to_db(
 
 
 def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str, process_id: str) -> None:
-    """Advance an existing queued plugin row after its OS process is persisted."""
+    """Advance an existing queued hook row after its OS process is persisted."""
     from archivebox.core.models import ArchiveResult
 
     started_at = parse_event_datetime(event.start_ts)
@@ -431,9 +273,9 @@ def mark_archiveresult_started(event: ProcessStartedEvent, *, snapshot_id: str,
     ArchiveResult.objects.filter(
         snapshot_id=snapshot_id,
         plugin=event.plugin_name,
+        hook_name=event.hook_name,
         status=ArchiveResult.StatusChoices.QUEUED,
     ).update(
-        hook_name=event.hook_name,
         status=ArchiveResult.StatusChoices.STARTED,
         start_ts=started_at,
         end_ts=None,
@@ -448,7 +290,7 @@ class ArchiveResultService(BaseService):
 
     def __init__(self, bus):
         self._completed_process_event_ids: set[str] = set()
-        self._save_locks: dict[tuple[str, str], asyncio.Lock] = {}
+        self._save_locks: dict[tuple[str, str, str], asyncio.Lock] = {}
         super().__init__(bus)
         self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
         self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
@@ -463,7 +305,7 @@ class ArchiveResultService(BaseService):
                 where=lambda candidate: self.bus.event_is_child_of(event, candidate),
             )
 
-        key = (str(event.snapshot_id), event.plugin)
+        key = (str(event.snapshot_id), event.plugin, event.hook_name)
         lock = self._save_locks.setdefault(key, asyncio.Lock())
         async with lock:
             await sync_to_async(_save_archiveresult_event_to_db, thread_sensitive=True)(event, process_started)

archivebox/services/process_service.py

@@ -43,10 +43,10 @@ def normalize_process_env(env: dict) -> dict:
         if is_sensitive_config_key(key) or (key in config_input_names and key not in allowed_config_keys):
             normalized.pop(key, None)
     if selected_plugins:
-        from abx_dl.models import discover_plugins, filter_plugins
         from archivebox.config.common import _plugin_enabled_config_keys
+        from archivebox.plugins.discovery import get_plugin_catalog
 
-        selected_plugins = set(filter_plugins(discover_plugins(runtime="archivebox"), sorted(selected_plugins), include_providers=True))
+        selected_plugins = set(get_plugin_catalog().select(sorted(selected_plugins)))
         for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
             normalized.setdefault(enabled_key, "True" if plugin_name in selected_plugins else "False")
     return normalized
@@ -100,6 +100,7 @@ class ProcessService(BaseService):
             process = await Process.objects.acreate(
                 machine=iface.machine,
                 iface=iface,
+                parent_id=None,
                 process_type=process_type,
                 worker_type=worker_type,
                 pwd=event.output_dir,
@@ -133,6 +134,7 @@ class ProcessService(BaseService):
             hook_path=event.hook_path,
         )
         await Process.objects.filter(id=process.id).aupdate(
+            parent_id=process.parent_id,
             pwd=process.pwd,
             cmd=process.cmd,
             env=process.env,
@@ -219,6 +221,7 @@ class ProcessService(BaseService):
             await Process.objects.acreate(
                 machine=iface.machine,
                 iface=iface,
+                parent_id=None,
                 process_type=process_type,
                 worker_type=worker_type,
                 pwd=event.output_dir,
@@ -239,6 +242,7 @@ class ProcessService(BaseService):
         updates = {
             "machine_id": iface.machine_id,
             "iface_id": iface.id,
+            "parent_id": process.parent_id,
             "pwd": event.output_dir,
             "env": process_env,
             "pid": event.pid or process.pid,

archivebox/services/runner.py

@@ -21,7 +21,6 @@ from rich.console import Console
 from rich.text import Text
 
 from abxpkg.binary_service import BinaryRequestEvent, BinaryService
-from abx_dl.config import GlobalConfig, RuntimeConfig
 from abx_dl.events import (
     CrawlAbortEvent,
     CrawlCleanupEvent,
@@ -29,21 +28,19 @@ from abx_dl.events import (
     CrawlEvent,
     CrawlSetupEvent,
     CrawlStartEvent,
-    MachineEvent,
     ProcessCompletedEvent,
     ProcessEvent,
     SnapshotCompletedEvent,
     SnapshotEvent,
     slow_warning_timeout,
 )
-from abx_dl.heartbeat import CrawlHeartbeat
 from abx_dl.limits import CrawlLimitState
-from abx_dl.models import Plugin, Snapshot as AbxSnapshot, discover_plugins, filter_plugins
+from abx_dl.catalog import PluginCatalog
+from abx_dl.models import Snapshot as AbxSnapshot
 from abx_dl.orchestrator import (
-    compute_phase_timeout,
+    ExecutionPlan,
     create_bus,
     install_plugins as abx_install_plugins,
-    setup_services as setup_abx_services,
 )
 from abx_dl.services.process_service import ProcessService as HookProcessService
 from abx_dl.services.snapshot_service import SnapshotService as HookSnapshotService
@@ -59,6 +56,7 @@ from archivebox.config.common import (
 )
 from archivebox.misc.db import run_db_analyze_batch
 from archivebox.core.shutdown_util import foreground_shutdown_signals, raise_if_shutdown_requested
+from archivebox.plugins.discovery import get_plugin_catalog
 from archivebox.search.sonic_daemon import register_sonic_daemon_event_handler
 from archivebox.workers.models import ACTIVE_STATE_LEASE_SECONDS
 from archivebox.crawls.locks import crawl_lifecycle_lock
@@ -118,8 +116,8 @@ def _runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: st
     Console(highlight=False).print(line)
 
 
-def _count_selected_hooks(plugins: dict[str, Plugin], selected_plugins: list[str] | None) -> int:
-    selected = filter_plugins(plugins, selected_plugins) if selected_plugins else plugins
+def _count_selected_hooks(catalog: PluginCatalog, selected_plugins: list[str] | None) -> int:
+    selected = catalog.select(selected_plugins) if selected_plugins else catalog
     return sum(1 for plugin in selected.values() for hook in plugin.hooks if "CrawlSetup" in hook.name or "Snapshot" in hook.name)
 
 
@@ -127,10 +125,6 @@ def _is_nonfatal_setup_hook(plugin_name: str, hook_name: str) -> bool:
     return plugin_name == "chrome" and hook_name.endswith("_chrome_kill_zombies")
 
 
-def _discover_archivebox_plugins() -> dict[str, Plugin]:
-    return discover_plugins(runtime="archivebox")
-
-
 def _runner_task_context() -> contextvars.Context:
     context = contextvars.copy_context()
     context.run(EventBus.current_event_context.set, None)
@@ -143,33 +137,6 @@ def _is_external_task_cancelled(error: asyncio.CancelledError) -> bool:
     return not isinstance(error, (EventHandlerAbortedError, EventHandlerCancelledError))
 
 
-async def _emit_machine_config(
-    bus,
-    *,
-    config: dict[str, Any],
-    derived_config: dict[str, Any],
-    parent_event=None,
-) -> None:
-    user_config = normalize_runtime_config(config)
-    user_config["ABX_RUNTIME"] = "archivebox"
-    derived_machine_config = normalize_runtime_config(derived_config)
-    user_event = MachineEvent(
-        config=user_config,
-        config_type="user",
-    )
-    if parent_event is not None:
-        user_event.event_parent_id = parent_event.event_id
-    await bus.emit(user_event).now()
-    if derived_machine_config:
-        derived_event = MachineEvent(
-            config=derived_machine_config,
-            config_type="derived",
-        )
-        if parent_event is not None:
-            derived_event.event_parent_id = parent_event.event_id
-        await bus.emit(derived_event).now()
-
-
 async def _run_event_now(event, timeout: float | None = None):
     await event.now(timeout=timeout)
     await event.wait(timeout=timeout)
@@ -217,7 +184,7 @@ class CrawlRunner:
     ):
         self.crawl = crawl
         self.bus = create_bus(name=_bus_name("ArchiveBox", str(crawl.id)), total_timeout=3600.0)
-        self.plugins = _discover_archivebox_plugins()
+        self.catalog = get_plugin_catalog()
         HookProcessService(self.bus, emit_jsonl=False, interactive_tty=interactive_interrupts)
         register_sonic_daemon_event_handler(self.bus)
         PersistedProcessService(self.bus)
@@ -284,10 +251,9 @@ class CrawlRunner:
         if self.allow_maintenance_on_inactive_crawl:
             # SEALED is the normal terminal state of a finished crawl, not a
             # cancellation signal for maintenance work on its already-sealed
-            # snapshots (search backend backfill, fs migration, etc.). When the
-            # runner is invoked with explicit snapshot_ids + selected_plugins,
-            # treat sealed as completed rather than cancelled so the requested
-            # maintenance hooks can actually run.
+            # snapshots (plugin backfill, fs migration, etc.). When the runner
+            # is given explicit snapshot_ids + selected_plugins, the caller has
+            # deliberately scoped work that remains valid after crawl sealing.
             return False
         return await Crawl.objects.filter(id=self.crawl.id, status=Crawl.StatusChoices.SEALED).aexists()
 
@@ -306,25 +272,14 @@ class CrawlRunner:
             await _run_event_now(abort_event, abort_event.event_timeout)
             return
 
-    def runtime_plugins(self) -> dict[str, Plugin]:
-        return filter_plugins(self.plugins, self.selected_plugins, include_providers=True) if self.selected_plugins else self.plugins
-
     @property
     def allow_maintenance_on_inactive_crawl(self) -> bool:
-        """Run targeted search indexing on an already-sealed snapshot.
-
-        This is the singular hook-execution exception to the unified lifecycle.
-        All other work must queue the Snapshot and Crawl normally.
-        """
+        """Run explicitly targeted plugin work on already-sealed snapshots."""
         return bool(
-            self.initial_snapshot_ids
-            and self.selected_plugins
-            and self.crawl.status == self.crawl.StatusChoices.SEALED
-            and all(plugin.startswith("search_backend_") for plugin in self.selected_plugins),
+            self.initial_snapshot_ids and self.selected_plugins and self.crawl.status == self.crawl.StatusChoices.SEALED,
         )
 
     async def run(self) -> None:
-        heartbeat: CrawlHeartbeat | None = None
         root_snapshot_id: str | None = None
         bus_destroyed = False
         try:
@@ -347,30 +302,17 @@ class CrawlRunner:
                 raise_on_first_signal=False,
             ):
                 snapshot_ids = await sync_to_async(self.load_run_state, thread_sensitive=True)()
-                heartbeat = CrawlHeartbeat(
-                    Path(self.crawl_output_dir),
-                    runtime="archivebox",
-                    crawl_id=str(self.crawl.id),
-                )
                 max_concurrent_snapshots = max(1, int(self.base_config.get("CRAWL_MAX_CONCURRENT_SNAPSHOTS", 1)))
                 self.max_concurrent_snapshots = max_concurrent_snapshots
                 self.snapshot_semaphore = asyncio.Semaphore(max_concurrent_snapshots)
                 live_ui = self._create_live_ui()
                 with live_ui if live_ui is not None else nullcontext():
                     try:
-                        await heartbeat.start()
-                        await _emit_machine_config(
-                            self.bus,
-                            config=self.base_config,
-                            derived_config=self.derived_config,
-                        )
                         if snapshot_ids:
                             root_snapshot_id = snapshot_ids[0]
                             await self.run_crawl(root_snapshot_id, snapshot_ids)
                     finally:
                         self._run_task = None
-                        if heartbeat is not None:
-                            await heartbeat.stop()
                         await self.stop_snapshot_tasks()
                         try:
                             await self.bus.wait_until_idle(timeout=1.0 if self._skip_wait_until_idle else 30.0)
@@ -382,8 +324,6 @@ class CrawlRunner:
         finally:
             if not bus_destroyed:
                 self._run_task = None
-                if heartbeat is not None:
-                    await heartbeat.stop()
                 await self.stop_snapshot_tasks()
                 await self.bus.destroy(clear=False)
             if self._live_stream is not None:
@@ -567,7 +507,7 @@ class CrawlRunner:
     def load_run_state(self) -> list[str]:
         from archivebox.config.common import get_config
         from archivebox.core.models import Snapshot
-        from archivebox.plugins.hooks import discover_hooks
+        from archivebox.plugins.discovery import get_enabled_plugins
         from archivebox.machine.models import Machine, NetworkInterface, Process
 
         self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
@@ -593,16 +533,28 @@ class CrawlRunner:
             if raw_plugins:
                 self.selected_plugins = [name.strip() for name in raw_plugins.split(",") if name.strip()]
             else:
+                enabled_plugins = get_enabled_plugins(config=self.base_config)
                 runtime_events = ("CrawlSetup", "CrawlCleanup", "Snapshot", "SnapshotCleanup")
                 runtime_plugins = {
-                    hook.parent.name for event_name in runtime_events for hook in discover_hooks(event_name, config=self.base_config)
+                    plugin.name for event_name in runtime_events for plugin, _hook in self.catalog.hooks(event_name, names=enabled_plugins)
                 }
                 self.selected_plugins = sorted(runtime_plugins) or None
         if self.crawl.is_paused:
             return []
+        system_task = self.crawl.get_system_task()
+        if system_task == "archivebox://update":
+            # Scheduled maintenance crawls are control-plane work, not URL
+            # input. The pre-unified crawl lifecycle delegated this sentinel
+            # directly to the database maintenance scan. Preserve that contract so
+            # the unified runner does not turn it into an archivebox://internal
+            # Snapshot and feed the literal sentinel through parser plugins.
+            from archivebox.cli.archivebox_update import process_all_db_snapshots
+
+            process_all_db_snapshots()
+            return []
         if self.initial_snapshot_ids:
             # Explicit ids select normal runnable work, except for the one
-            # sealed-search backfill admitted by allow_maintenance_on_inactive_crawl.
+            # targeted maintenance admitted by allow_maintenance_on_inactive_crawl.
             return [str(snapshot_id) for snapshot_id in self.initial_snapshot_ids]
         pending_snapshots = list(
             self.crawl.snapshot_set.filter(status__in=Snapshot.RUNNABLE_STATES)
@@ -695,7 +647,7 @@ class CrawlRunner:
         if crawl.is_finished():
             if crawl.status != Crawl.StatusChoices.SEALED:
                 if crawl.status == Crawl.StatusChoices.STARTED:
-                    crawl.sm.seal()
+                    crawl.seal()
                 else:
                     crawl.update_and_requeue(
                         status=Crawl.StatusChoices.SEALED,
@@ -751,10 +703,10 @@ class CrawlRunner:
                 "LINES": str(terminal_height),
             },
         )
-        plugins_label = ", ".join(self.selected_plugins) if self.selected_plugins else f"all ({len(self.plugins)} available)"
+        plugins_label = ", ".join(self.selected_plugins) if self.selected_plugins else f"all ({len(self.catalog)} available)"
         live_ui = LiveBusUI(
             self.bus,
-            total_hooks=_count_selected_hooks(self.plugins, self.selected_plugins),
+            total_hooks=_count_selected_hooks(self.catalog, self.selected_plugins),
             timeout_seconds=self.base_config["TIMEOUT"],
             ui_console=ui_console,
             interactive_tty=interactive_tty,
@@ -811,7 +763,7 @@ class CrawlRunner:
         normalized_config = normalize_runtime_config(config)
         configured_plugins = [name.strip().lower() for name in str(normalized_config.get("PLUGINS") or "").split(",") if name.strip()]
         if configured_plugins:
-            selected_plugin_names = set(filter_plugins(self.plugins, configured_plugins, include_providers=True))
+            selected_plugin_names = set(self.catalog.select(configured_plugins))
             for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
                 normalized_config.setdefault(enabled_key, plugin_name in selected_plugin_names)
         return {
@@ -839,18 +791,23 @@ class CrawlRunner:
         config = normalize_runtime_config(snapshot["config"])
         derived_config = normalize_runtime_config(self.derived_config)
         output_dir = Path(self.crawl_output_dir)
-        plugins = self.runtime_plugins()
+        plan = ExecutionPlan.build(
+            self.catalog,
+            selected_plugins=self.selected_plugins,
+            config=config,
+            derived_config=derived_config,
+            runtime="archivebox",
+        )
+        setup_hooks = [(plugin, hook) for plugin in plan.catalog.values() for hook in plugin.filter_hooks("CrawlSetup")]
         abx_snapshot = AbxSnapshot(
             id=snapshot["id"],
             url=snapshot["url"],
             depth=int(snapshot["depth"]),
             crawl_id=str(self.crawl.id),
         )
-        setup_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("CrawlSetup")]
-        crawl_setup_phase_timeout = compute_phase_timeout(setup_hooks, config)
-        snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")]
+        crawl_setup_phase_timeout = plan.crawl_setup_timeout
         max_snapshot_count = max(1, int(config.get("CRAWL_MAX_URLS") or len(snapshot_ids) or 1))
-        snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) + 120.0
+        snapshot_phase_timeout = plan.snapshot_timeout + 120.0
         all_snapshots_phase_timeout = snapshot_phase_timeout * max_snapshot_count
         crawl_cleanup_phase_timeout = crawl_setup_phase_timeout
         crawl_lifecycle_timeout = (
@@ -860,10 +817,19 @@ class CrawlRunner:
             + CrawlCompletedEvent.model_fields["event_timeout"].default
             + 30.0
         )
-        await _emit_machine_config(self.bus, config=config, derived_config=derived_config)
-        setup_abx_services(
+        if plan.catalog:
+            await abx_install_plugins(
+                plan,
+                output_dir=output_dir,
+                emit_jsonl=False,
+                bus=self.bus,
+                BinaryService=None,
+                ProcessService=None,
+            )
+        else:
+            await plan.seed_config(self.bus)
+        plan.attach_services(
             self.bus,
-            plugins=plugins,
             url=snapshot["url"],
             snapshot=abx_snapshot,
             output_dir=output_dir,
@@ -874,14 +840,11 @@ class CrawlRunner:
             snapshot_cleanup_enabled=False,
             crawl_cleanup_enabled=True,
             crawl_completed_enabled=False,
-            crawl_setup_phase_timeout=crawl_setup_phase_timeout,
-            snapshot_phase_timeout=crawl_setup_phase_timeout,
-            snapshot_cleanup_phase_timeout=crawl_setup_phase_timeout,
-            crawl_cleanup_phase_timeout=crawl_setup_phase_timeout,
             auto_install=True,
             emit_jsonl=False,
             abort_requested=self.crawl_is_cancelled,
             PluginBinariesService=None,
+            PluginBinaryEnvService=None,
             BinaryService=None,
             ProcessService=None,
             ArchiveResultService=None,
@@ -1040,12 +1003,10 @@ class CrawlRunner:
             def queued_plugins_selected_by_config(queued_plugins: list[str]) -> list[str]:
                 if not snapshot_selected_plugins:
                     return queued_plugins
-                expanded_selected_plugins = set(
-                    filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True).keys(),
-                )
+                expanded_selected_plugins = set(self.catalog.select(snapshot_selected_plugins))
                 return [plugin for plugin in queued_plugins if plugin in expanded_selected_plugins]
 
-            queued_plugins = None
+            selected_hooks_by_plugin = None
             if snapshot["status"] == "started":
                 _reset_count, running_count = await sync_to_async(snapshot["_snapshot"].reset_abandoned_results, thread_sensitive=True)()
                 if running_count:
@@ -1063,8 +1024,8 @@ class CrawlRunner:
                     )
                     return
                 if not self.selected_plugins_from_args:
-                    queued_plugins = await sync_to_async(
-                        queued_plugins_for_snapshot,
+                    queued_plugins, selected_hooks_by_plugin = await sync_to_async(
+                        queued_plugins_and_hooks_for_snapshot,
                         thread_sensitive=True,
                     )(snapshot["id"])
                     if queued_plugins:
@@ -1077,10 +1038,13 @@ class CrawlRunner:
                                     snapshot["id"],
                                     disabled_queued_plugins,
                                 )
+                            selected_hooks_by_plugin = {
+                                plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
+                            }
                         snapshot_selected_plugins = queued_plugins
             elif not self.selected_plugins_from_args:
-                queued_plugins = await sync_to_async(
-                    queued_plugins_for_snapshot,
+                queued_plugins, selected_hooks_by_plugin = await sync_to_async(
+                    queued_plugins_and_hooks_for_snapshot,
                     thread_sensitive=True,
                 )(snapshot["id"])
                 if queued_plugins:
@@ -1093,6 +1057,9 @@ class CrawlRunner:
                                 snapshot["id"],
                                 disabled_queued_plugins,
                             )
+                        selected_hooks_by_plugin = {
+                            plugin: hooks for plugin, hooks in (selected_hooks_by_plugin or {}).items() if plugin in queued_plugins
+                        }
                     snapshot_selected_plugins = queued_plugins
             if snapshot["depth"] > 0 and CrawlLimitState.from_config(snapshot["config"]).get_stop_reason() in (
                 "crawl_max_size",
@@ -1102,15 +1069,11 @@ class CrawlRunner:
                 return
             derived_config = normalize_runtime_config(self.derived_config)
             output_dir = Path(snapshot["output_dir"])
-            plugins = (
-                filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
-                if snapshot_selected_plugins
-                else self.plugins
-            )
-            if queued_plugins is not None:
-                await sync_to_async(fail_unavailable_queued_plugins, thread_sensitive=True)(
+            plugins = self.catalog.select(snapshot_selected_plugins) if snapshot_selected_plugins else self.catalog
+            if selected_hooks_by_plugin is not None:
+                await sync_to_async(fail_unavailable_queued_hooks, thread_sensitive=True)(
                     snapshot["id"],
-                    queued_plugins,
+                    selected_hooks_by_plugin,
                     plugins,
                 )
                 remaining_queued_plugins = await sync_to_async(
@@ -1129,31 +1092,34 @@ class CrawlRunner:
                         await sync_to_async(run_snapshot_maintenance, thread_sensitive=True)(snapshot_id, output_dir=output_dir)
                     return
                 snapshot_selected_plugins = remaining_queued_plugins
-                plugins = filter_plugins(self.plugins, snapshot_selected_plugins, include_providers=True)
+                plugins = self.catalog.select(snapshot_selected_plugins)
+                selected_hooks_by_plugin = include_background_prerequisite_hooks(selected_hooks_by_plugin, plugins)
             abx_snapshot = AbxSnapshot(
                 id=snapshot["id"],
                 url=snapshot["url"],
                 depth=int(snapshot["depth"]),
                 crawl_id=str(self.crawl.id),
             )
-            snapshot_hooks = [(plugin, hook) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot")]
-            snapshot_phase_timeout = compute_phase_timeout(snapshot_hooks, config) + 120.0
-            await _emit_machine_config(self.bus, config=config, derived_config=derived_config, parent_event=crawl_start_event)
-            snapshot_service = HookSnapshotService(
+            plan = ExecutionPlan.build(
+                self.catalog,
+                selected_plugins=snapshot_selected_plugins,
+                config=config,
+                derived_config=derived_config,
+                runtime="archivebox",
+            )
+            plugins = plan.catalog
+            snapshot_phase_timeout = plan.snapshot_timeout + 120.0
+            await plan.seed_config(self.bus, parent_event=crawl_start_event)
+            snapshot_service = plan.attach_snapshot_service(
                 self.bus,
                 url=snapshot["url"],
                 snapshot=abx_snapshot,
                 output_dir=output_dir,
-                plugins=plugins,
-                config=RuntimeConfig(
-                    user=GlobalConfig(**{**config, "ABX_RUNTIME": "archivebox"}),
-                    derived=derived_config,
-                ),
-                snapshot_phase_timeout=snapshot_phase_timeout,
-                snapshot_cleanup_enabled=True,
-                snapshot_cleanup_phase_timeout=snapshot_phase_timeout,
+                snapshot_service=HookSnapshotService,
+                timeout_padding=120.0,
                 abort_requested=self.crawl_is_cancelled,
-                selected_hooks_by_plugin=None,
+                selected_hooks_by_plugin=selected_hooks_by_plugin,
+                emit_discovered_snapshot_events=False,
             )
             try:
                 snapshot_event = SnapshotEvent(
@@ -1181,7 +1147,7 @@ class CrawlRunner:
                 # SnapshotCompletedEvent is the normal projection path, but the
                 # runner is the scheduler owner. Finalize idempotently here too
                 # so a completed snapshot cannot remain STARTED if the event was
-                # observed before its DB projector advanced the state machine.
+                # observed before its DB projector advanced the lifecycle.
                 crawl_limit_stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
                 await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(
                     snapshot_id,
@@ -1194,18 +1160,8 @@ class CrawlRunner:
                 await self.enqueue_discovered_snapshots_from_outputs(snapshot)
 
                 def _seal_when_last_snapshot_finished() -> None:
-                    # run_snapshot replaces self.crawl per-snapshot, so multiple
-                    # concurrent tasks each load a fresh Crawl/SM pointing at the
-                    # same DB row. The "no open snapshots" check is non-atomic
-                    # with sm.seal(), so two tasks racing to finish the last
-                    # snapshot can both pass the guard. The first call drives
-                    # the SM to a final state (engine.running=False); the loser
-                    # then raises TransitionNotAllowed even though current_state
-                    # still reads STARTED off its stale model field. Re-read the
-                    # row right before the call and swallow the race so the
-                    # task that lost the lap doesn't fail the whole snapshot.
-                    from statemachine.exceptions import TransitionNotAllowed
-
+                    # Re-read immediately before the idempotent conditional
+                    # update so concurrent last-snapshot completions are safe.
                     crawl = self.crawl
                     crawl.refresh_from_db(fields=["status"])
                     if crawl.status != crawl.StatusChoices.STARTED:
@@ -1214,12 +1170,7 @@ class CrawlRunner:
                         status__in=crawl.snapshot_set.model.OPEN_STATES,
                     ).exists():
                         return
-                    try:
-                        crawl.sm.seal()
-                    except TransitionNotAllowed:
-                        # Another task sealed it between our refresh and the
-                        # SM call. Idempotent by design.
-                        pass
+                    crawl.seal()
 
                 await sync_to_async(_seal_when_last_snapshot_finished, thread_sensitive=True)()
             finally:
@@ -1233,7 +1184,7 @@ class CrawlRunner:
             return
         # Limit stops are runner-owned cancellation decisions, not normal
         # "all ArchiveResults finished" lifecycle seals. Updating the row
-        # directly avoids racing the state machine's in-memory state while
+        # directly avoids racing a concurrent lifecycle update while
         # concurrent snapshot tasks are stopping because the crawl-wide limit
         # has already been reached.
         snapshot.update_and_requeue(
@@ -1323,7 +1274,6 @@ async def _run_binary(binary_id: str) -> None:
     from archivebox.machine.models import Binary, Machine
 
     binary = await Binary.objects.aget(id=binary_id)
-    plugins = _discover_archivebox_plugins()
     config = get_config(include_machine=False)
     machine = await sync_to_async(Machine.current, thread_sensitive=True)()
     derived_config = normalize_runtime_config(machine.config)
@@ -1336,9 +1286,16 @@ async def _run_binary(binary_id: str) -> None:
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)
-    setup_abx_services(
+    catalog = get_plugin_catalog()
+    plan = ExecutionPlan.build(
+        catalog,
+        selected_plugins=list(catalog),
+        config=config,
+        derived_config=derived_config,
+        runtime="archivebox",
+    )
+    plan.attach_services(
         bus,
-        plugins=plugins,
         install_enabled=False,
         crawl_setup_enabled=False,
         crawl_start_enabled=False,
@@ -1348,7 +1305,7 @@ async def _run_binary(binary_id: str) -> None:
         emit_jsonl=False,
         BinaryService=None,
     )
-    await _emit_machine_config(bus, config=config, derived_config=derived_config)
+    await plan.seed_config(bus)
 
     try:
         await bus.emit(
@@ -1375,56 +1332,82 @@ def run_binary(binary_id: str) -> None:
     asyncio.run(_run_binary(binary_id))
 
 
-def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
+def queued_plugins_and_hooks_for_snapshot(snapshot_id: str) -> tuple[list[str] | None, dict[str, set[str] | None] | None]:
     from archivebox.core.models import ArchiveResult
 
-    queued_plugins = list(
+    queued_results = list(
         ArchiveResult.objects.filter(
             snapshot_id=snapshot_id,
             status=ArchiveResult.StatusChoices.QUEUED,
         )
         .exclude(plugin="")
-        .order_by("plugin")
-        .values_list("plugin", flat=True),
+        .only("id", "plugin", "hook_name"),
     )
-    return queued_plugins or None
+
+    selected_hooks_by_plugin: dict[str, set[str] | None] = {}
+    queued_plugins = sorted({result.plugin for result in queued_results})
+    for result in queued_results:
+        # hook_name is the modern scheduler identity. Empty hook_name rows are
+        # legacy plugin-level work and must keep running the whole plugin.
+        if not result.hook_name:
+            selected_hooks_by_plugin[result.plugin] = None
+        elif result.plugin not in selected_hooks_by_plugin:
+            selected_hooks_by_plugin[result.plugin] = {result.hook_name}
+        elif selected_hooks_by_plugin[result.plugin] is not None:
+            selected_hooks_by_plugin[result.plugin].add(result.hook_name)
+    if queued_plugins:
+        return queued_plugins, selected_hooks_by_plugin
+    return None, None
+
+
+def queued_plugins_for_snapshot(snapshot_id: str) -> list[str] | None:
+    queued_plugins, _selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(snapshot_id)
+    return queued_plugins
 
 
 def config_overrides_for_queued_plugins(selected_plugins: list[str], **overrides: Any) -> dict[str, Any]:
     config_overrides = dict(overrides)
     config_overrides["PLUGINS"] = ",".join(selected_plugins)
-    selected_plugin_names = set(
-        filter_plugins(_discover_archivebox_plugins(), [plugin_name.lower() for plugin_name in selected_plugins], include_providers=True),
-    )
+    selected_plugin_names = set(get_plugin_catalog().select(plugin_name.lower() for plugin_name in selected_plugins))
     for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
         config_overrides[enabled_key] = plugin_name in selected_plugin_names
-    for plugin_name in selected_plugins:
-        if plugin_name.startswith("search_backend_"):
-            config_overrides[f"{plugin_name.upper()}_ENABLED"] = True
     return config_overrides
 
 
-def fail_unavailable_queued_plugins(
+def fail_unavailable_queued_hooks(
     snapshot_id: str,
-    queued_plugins: list[str],
-    plugins: dict[str, Plugin],
+    selected_hooks_by_plugin: dict[str, set[str] | None],
+    plugins: PluginCatalog,
 ) -> None:
     from archivebox.core.models import ArchiveResult
 
-    unavailable_plugins = set(queued_plugins) - set(plugins)
-    if not unavailable_plugins:
-        return
     now = timezone.now()
-    ArchiveResult.objects.filter(
-        snapshot_id=snapshot_id,
-        plugin__in=unavailable_plugins,
-        status=ArchiveResult.StatusChoices.QUEUED,
-    ).update(
-        status=ArchiveResult.StatusChoices.FAILED,
-        start_ts=now,
-        end_ts=now,
-        output_str="Queued plugin is not available in this ArchiveBox installation",
-    )
+    for plugin_name, selected_hook_names in selected_hooks_by_plugin.items():
+        if selected_hook_names is None:
+            continue
+        if plugin_name in plugins:
+            available_hook_names = {
+                name for hook in plugins[plugin_name].filter_hooks("Snapshot") for name in (hook.name, Path(hook.name).stem)
+            }
+        else:
+            available_hook_names = set()
+        missing_hook_names = [hook_name for hook_name in selected_hook_names if hook_name not in available_hook_names]
+        if not missing_hook_names:
+            continue
+        # Hook-level resume rows are durable scheduler state. If a plugin is
+        # installed but no longer exposes a queued hook, mark that row failed so
+        # the snapshot is not retried forever with no hook left to execute.
+        ArchiveResult.objects.filter(
+            snapshot_id=snapshot_id,
+            plugin=plugin_name,
+            hook_name__in=missing_hook_names,
+            status=ArchiveResult.StatusChoices.QUEUED,
+        ).update(
+            status=ArchiveResult.StatusChoices.FAILED,
+            start_ts=now,
+            end_ts=now,
+            output_str="Queued hook is no longer available in the installed plugin",
+        )
 
 
 def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> None:
@@ -1449,6 +1432,35 @@ def skip_disabled_queued_plugins(snapshot_id: str, plugin_names: list[str]) -> N
     )
 
 
+def include_background_prerequisite_hooks(
+    selected_hooks_by_plugin: dict[str, set[str] | None],
+    plugins: PluginCatalog,
+) -> dict[str, set[str] | None]:
+    expanded: dict[str, set[str] | None] = {}
+    for plugin_name, selected_hook_names in selected_hooks_by_plugin.items():
+        if selected_hook_names is None or plugin_name not in plugins:
+            expanded[plugin_name] = selected_hook_names
+            continue
+        plugin_hooks = sorted(plugins[plugin_name].filter_hooks("Snapshot"), key=lambda hook: hook.sort_key)
+        selected_sort_keys = [
+            hook.sort_key for hook in plugin_hooks if hook.name in selected_hook_names or Path(hook.name).stem in selected_hook_names
+        ]
+        if not selected_sort_keys:
+            expanded[plugin_name] = set(selected_hook_names)
+            continue
+        first_selected_sort_key = min(selected_sort_keys)
+        expanded_hook_names = set(selected_hook_names)
+        # Earlier background hooks publish live resources (e.g. Chrome tabs)
+        # needed by later foreground hooks, but completed foreground hooks stay
+        # final and are not rerun during hook-level resume.
+        for hook in plugin_hooks:
+            if hook.is_background and hook.sort_key < first_selected_sort_key:
+                expanded_hook_names.add(hook.name)
+                expanded_hook_names.add(Path(hook.name).stem)
+        expanded[plugin_name] = expanded_hook_names
+    return expanded
+
+
 def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]:
     from archivebox.config.common import get_config
     from archivebox.core.models import Snapshot
@@ -1459,11 +1471,8 @@ def snapshot_hooks_for_pending_archiveresults(snapshot) -> list[tuple[str, str]]
     crawl_plugin_names = [name.strip() for name in str((snapshot.crawl.config or {}).get("PLUGINS") or "").split(",") if name.strip()]
     config_plugin_names = [name.strip() for name in str(config.PLUGINS or "").split(",") if name.strip()]
     plugin_names = snapshot_plugin_names or crawl_plugin_names or config_plugin_names or get_enabled_plugins(config=config)
-    plugins = (
-        filter_plugins(_discover_archivebox_plugins(), plugin_names, include_providers=True)
-        if plugin_names
-        else _discover_archivebox_plugins()
-    )
+    catalog = get_plugin_catalog()
+    plugins = catalog.select(plugin_names) if plugin_names else catalog
     if snapshot.url == Snapshot.INTERNAL_INPUT_URL:
         plugins = {name: plugin for name, plugin in plugins.items() if getattr(plugin.config, "x_accepts_internal_input", False)}
     return sorted((plugin.name, hook.name) for plugin in plugins.values() for hook in plugin.filter_hooks("Snapshot"))
@@ -1483,8 +1492,8 @@ def run_snapshot_maintenance(snapshot_id: str, *, output_dir: Path | None = None
     # The only thing queued work changes is the next scheduler value:
     # - open lifecycle rows: leave them due for extraction after maintenance
     # - no queued work left on a sealed row: clear retry_at
-    # - queued rows remain on a sealed Snapshot: leave it due so the search
-    #   backfill exception can process them on the next tick
+    # - queued rows remain on a sealed Snapshot: leave it due so targeted
+    #   plugin maintenance can process them on the next tick
     current_retry_at = snapshot.retry_at
     next_retry_at = timezone.now() if has_queued_results or snapshot.status in Snapshot.OPEN_STATES else None
     snapshot.retry_at = next_retry_at
@@ -1553,7 +1562,7 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
                 if not crawl.claim_processing_lock(lock_seconds=lock_seconds):
                     return False
                 crawl.refresh_from_db()
-                crawl.sm.tick()
+                crawl.advance_lifecycle()
                 return True
 
             # retry_at is the only queue/ownership signal the runner sees.
@@ -1588,7 +1597,7 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
             return False
         crawl.refresh_from_db()
         if crawl.status == crawl.StatusChoices.STARTED and crawl.is_finished():
-            crawl.sm.tick()
+            crawl.advance_lifecycle()
             return True
         _runner_console_line(crawl=crawl)
         run_crawl(str(crawl.id), process_discovered_snapshots_inline=True, interactive_interrupts=interactive_interrupts)
@@ -1598,7 +1607,7 @@ def _run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: b
         if not type(crawl).claim_for_worker(crawl, lock_seconds=lock_seconds):
             return False
         _runner_console_line(crawl=crawl, status="SEALED")
-        crawl.cleanup()
+        crawl.cleanup_runtime()
         crawl.update_and_requeue(retry_at=None)
         return True
 
@@ -1633,7 +1642,7 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
 
     if snapshot.is_paused:
         # Paused work never executes out of band. Preserve the lifecycle marker
-        # until an explicit resume moves it through the normal state machine.
+        # until an explicit resume moves it through the normal lifecycle.
         from archivebox.core.models import ArchiveResult
 
         ArchiveResult.pause_queryset(snapshot.archiveresult_set.all())
@@ -1655,48 +1664,30 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
             snapshot.refresh_from_db()
         selected_plugins = queued_plugins_for_snapshot(str(snapshot.id))
         if selected_plugins:
-            search_only_plugins = all(plugin.startswith("search_backend_") for plugin in selected_plugins)
-            if not search_only_plugins:
-                snapshot.update_and_requeue(
-                    status=Snapshot.StatusChoices.QUEUED,
-                    retry_at=timezone.now(),
-                    current_step=0,
-                )
-                snapshot.refresh_from_db()
-            else:
-                _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
-                run_crawl(
-                    str(snapshot.crawl_id),
-                    snapshot_ids=[str(snapshot.id)],
-                    selected_plugins=selected_plugins,
-                    process_discovered_snapshots_inline=True,
-                    interactive_interrupts=interactive_interrupts,
-                    config_overrides=config_overrides_for_queued_plugins(selected_plugins),
-                    selected_plugins_are_explicit=False,
-                )
-                from archivebox.core.models import ArchiveResult
+            _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot)
+            run_crawl(
+                str(snapshot.crawl_id),
+                snapshot_ids=[str(snapshot.id)],
+                selected_plugins=selected_plugins,
+                process_discovered_snapshots_inline=True,
+                interactive_interrupts=interactive_interrupts,
+                config_overrides=config_overrides_for_queued_plugins(selected_plugins),
+                selected_plugins_are_explicit=False,
+            )
+            from archivebox.core.models import ArchiveResult
 
-                has_queued_results = ArchiveResult.objects.filter(
-                    snapshot_id=snapshot.id,
-                    status=ArchiveResult.StatusChoices.QUEUED,
-                ).exists()
-                if not has_queued_results:
-                    type(snapshot).objects.filter(
-                        pk=snapshot.pk,
-                        status=snapshot.StatusChoices.SEALED,
-                    ).update(
-                        retry_at=None,
-                        modified_at=timezone.now(),
-                    )
-                else:
-                    type(snapshot).objects.filter(
-                        pk=snapshot.pk,
-                        status=snapshot.StatusChoices.SEALED,
-                    ).update(
-                        retry_at=timezone.now(),
-                        modified_at=timezone.now(),
-                    )
-                return True
+            has_queued_results = ArchiveResult.objects.filter(
+                snapshot_id=snapshot.id,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).exists()
+            type(snapshot).objects.filter(
+                pk=snapshot.pk,
+                status=snapshot.StatusChoices.SEALED,
+            ).update(
+                retry_at=timezone.now() if has_queued_results else None,
+                modified_at=timezone.now(),
+            )
+            return True
         if snapshot.status == Snapshot.StatusChoices.SEALED:
             if maintenance_ran:
                 return True
@@ -1712,10 +1703,10 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
         return False
     snapshot.refresh_from_db()
     if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        has_server_workset = snapshot.archiveresult_set.exclude(
+        has_server_archiveresults = snapshot.archiveresult_set.exclude(
             hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
         ).exists()
-        if has_server_workset and snapshot.is_finished_processing():
+        if has_server_archiveresults and snapshot.is_finished_processing():
             finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
             snapshot.refresh_from_db()
             if snapshot.status == Snapshot.StatusChoices.SEALED:
@@ -1724,12 +1715,12 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
                     snapshot.refresh_from_db()
                 _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
                 return True
-        if not has_server_workset:
-            # Materialize one durable row per configured plugin. Existing
-            # browser-uploaded rows are reused and queued so the server adds its
-            # outputs to that plugin result instead of creating a sibling row.
+        # A Snapshot with no server hook rows is fresh lifecycle work; materialize
+        # its configured hook set. Browser-extension uploads are completed external
+        # outputs, not the durable server workset, and must coexist with these rows.
+        if not has_server_archiveresults:
             snapshot.create_pending_archiveresults(hooks=snapshot_hooks_for_pending_archiveresults(snapshot))
-        snapshot.sm.tick()
+        snapshot.advance_lifecycle()
         snapshot.refresh_from_db()
         if snapshot.status == Snapshot.StatusChoices.SEALED:
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
@@ -1744,12 +1735,12 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
             _runner_console_line(crawl_id=snapshot.crawl_id, snapshot=snapshot, status="SEALED")
             return True
     if snapshot.status == Snapshot.StatusChoices.STARTED:
-        queued_plugins = queued_plugins_for_snapshot(str(snapshot.id))
-        if queued_plugins:
-            fail_unavailable_queued_plugins(
+        queued_plugins, selected_hooks_by_plugin = queued_plugins_and_hooks_for_snapshot(str(snapshot.id))
+        if queued_plugins and selected_hooks_by_plugin:
+            fail_unavailable_queued_hooks(
                 str(snapshot.id),
-                queued_plugins,
-                _discover_archivebox_plugins(),
+                selected_hooks_by_plugin,
+                get_plugin_catalog(),
             )
             if not queued_plugins_for_snapshot(str(snapshot.id)):
                 finalize_completed_snapshot(str(snapshot.id), output_dir=Path(snapshot.output_dir))
@@ -1772,10 +1763,10 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
     )
     snapshot.refresh_from_db()
     if queued_plugins_for_snapshot(str(snapshot.id)):
-        # Plugin-level resume work is tracked by queued ArchiveResult rows, not by
+        # Hook-level resume work is tracked by queued ArchiveResult rows, not by
         # the Snapshot lease. If a partial pass returns with rows still queued,
         # wake the Snapshot immediately so takeover does not wait out a stale
-        # active-state lock before running the remaining plugins.
+        # active-state lock before running the remaining hooks.
         snapshot.update_and_requeue(retry_at=timezone.now())
     return True
 
@@ -1798,7 +1789,7 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     from archivebox.machine.models import Machine
     from archivebox.plugins.discovery import get_enabled_plugins
 
-    plugins = _discover_archivebox_plugins()
+    catalog = get_plugin_catalog()
     config = get_config(include_machine=False)
     machine = await sync_to_async(Machine.current, thread_sensitive=True)()
     derived_config = normalize_runtime_config(machine.config)
@@ -1811,18 +1802,28 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
     TagService(bus)
     ArchiveResultService(bus)
     MachineService(bus)
-    await _emit_machine_config(bus, config=config, derived_config=derived_config)
     live_stream = None
     bus_destroyed = False
 
     try:
         if plugin_names:
-            selected_plugins = filter_plugins(plugins, list(plugin_names), include_providers=True)
+            selected_plugins = catalog.select(plugin_names)
         else:
-            selected_plugins = filter_plugins(plugins, get_enabled_plugins(config=config), include_providers=True)
+            selected_plugins = catalog.select(get_enabled_plugins(config=config))
         if not selected_plugins:
             return
-        plugins_label = ", ".join(plugin_names) if plugin_names else f"enabled ({len(selected_plugins)} of {len(plugins)} available)"
+        plugins_label = ", ".join(plugin_names) if plugin_names else f"enabled ({len(selected_plugins)} of {len(catalog)} available)"
+        install_config = dict(config)
+        for plugin in selected_plugins.values():
+            if plugin.enabled_key in plugin.config.properties:
+                install_config[plugin.enabled_key] = True
+        plan = ExecutionPlan.build(
+            catalog,
+            selected_plugins=list(selected_plugins),
+            config=install_config,
+            derived_config=derived_config,
+            runtime="archivebox",
+        )
         timeout_seconds = config["TIMEOUT"]
         stdout_is_tty = sys.stdout.isatty()
         stderr_is_tty = sys.stderr.isatty()
@@ -1875,11 +1876,8 @@ async def _run_install(plugin_names: list[str] | None = None) -> None:
             with live_ui if live_ui is not None else nullcontext():
                 try:
                     await abx_install_plugins(
-                        plugin_names=selected_plugins,
-                        plugins=plugins,
+                        plan,
                         output_dir=output_dir,
-                        config_overrides=config,
-                        derived_config_overrides=derived_config,
                         emit_jsonl=False,
                         bus=bus,
                         BinaryService=None,
@@ -2050,23 +2048,22 @@ def _run_due_queued_plugin_result(
         config_overrides=config_overrides_for_queued_plugins(selected_plugins, CRAWL_MAX_CONCURRENT_SNAPSHOTS=batch_size),
         selected_plugins_are_explicit=False,
     )
-    if all(plugin.startswith("search_backend_") for plugin in selected_plugins):
-        queued_results = ArchiveResult.objects.filter(
-            snapshot_id=OuterRef("pk"),
-            status=ArchiveResult.StatusChoices.QUEUED,
-            plugin__in=selected_plugins,
-        )
-        Snapshot.objects.filter(
-            id__in=claimed_snapshot_ids,
-            status=Snapshot.StatusChoices.SEALED,
-        ).annotate(
-            has_queued_results=Exists(queued_results),
-        ).filter(
-            has_queued_results=False,
-        ).update(
-            retry_at=None,
-            modified_at=timezone.now(),
-        )
+    queued_results = ArchiveResult.objects.filter(
+        snapshot_id=OuterRef("pk"),
+        status=ArchiveResult.StatusChoices.QUEUED,
+        plugin__in=selected_plugins,
+    )
+    Snapshot.objects.filter(
+        id__in=claimed_snapshot_ids,
+        status=Snapshot.StatusChoices.SEALED,
+    ).annotate(
+        has_queued_results=Exists(queued_results),
+    ).filter(
+        has_queued_results=False,
+    ).update(
+        retry_at=None,
+        modified_at=timezone.now(),
+    )
     return True
 
 
@@ -2100,7 +2097,7 @@ def run_pending_crawls(
     from archivebox.config.common import get_config
     from archivebox.crawls.models import Crawl, CrawlSchedule
     from archivebox.core.models import ArchiveResult, Snapshot
-    from archivebox.plugins.hooks import discover_hooks
+    from archivebox.plugins.discovery import get_enabled_plugins, get_plugin_catalog
     from archivebox.machine.models import Process
 
     crawl_claim_lock_seconds = 10
@@ -2232,19 +2229,17 @@ def run_pending_crawls(
             ):
                 continue
 
-        # Search backend selection is live crawl-execution config, not an
-        # installed-plugin list. Old queued rows for a backend that is disabled
-        # by the current Machine/Crawl/Snapshot config must remain queued so
-        # they can run if the user re-enables that backend, but they should not
-        # launch a standalone hook process just to skip after imports/config
-        # hydration. Refreshing here preserves mid-run config edits while using
-        # the same enabled-hook discovery path that created ArchiveResult rows.
+        # Plugin selection is live crawl-execution config, not an installed-
+        # plugin list. Old queued rows for a plugin that is disabled by the
+        # current Machine/Crawl/Snapshot config remain queued until the user
+        # re-enables it. Refresh here to preserve mid-run config edits while
+        # using the same enabled-hook discovery path that created the rows.
         runtime_config = get_config()
-        search_plugin_names = frozenset(
-            hook.parent.name for hook in discover_hooks("Snapshot", config=runtime_config) if hook.parent.name.startswith("search_backend_")
-        )
+        catalog = get_plugin_catalog()
+        enabled_plugins = get_enabled_plugins(config=runtime_config)
+        queued_plugin_names = frozenset(plugin.name for plugin, _hook in catalog.hooks("Snapshot", names=enabled_plugins))
         if _run_due_queued_plugin_result(
-            search_plugin_names,
+            queued_plugin_names,
             crawl_id=crawl_id,
             lock_seconds=60,
             interactive_interrupts=interactive_interrupts,
@@ -2262,13 +2257,13 @@ def run_pending_crawls(
                 retry_at__lte=timezone.now(),
                 status=Snapshot.StatusChoices.SEALED,
             )
-            if search_plugin_names:
-                queued_search_snapshot_ids = ArchiveResult.objects.filter(
+            if queued_plugin_names:
+                queued_plugin_snapshot_ids = ArchiveResult.objects.filter(
                     status=ArchiveResult.StatusChoices.QUEUED,
-                    plugin__in=search_plugin_names,
+                    plugin__in=queued_plugin_names,
                 ).values("snapshot_id")
                 sealed_snapshots = sealed_snapshots.exclude(
-                    id__in=queued_search_snapshot_ids,
+                    id__in=queued_plugin_snapshot_ids,
                 )
             if crawl_id:
                 sealed_snapshots = sealed_snapshots.filter(crawl_id=crawl_id)

archivebox/services/snapshot_service.py

@@ -74,10 +74,10 @@ def finalize_completed_snapshot(
         )
 
     if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        snapshot.sm.tick()
+        snapshot.advance_lifecycle()
         snapshot.refresh_from_db()
     if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.is_finished_processing():
-        snapshot.sm.seal()
+        snapshot.seal()
         snapshot.refresh_from_db()
 
     snapshot.write_index_jsonl(output_dir=output_dir)
@@ -119,7 +119,7 @@ class SnapshotService(BaseService):
                     hooks = await sync_to_async(snapshot_hooks_for_pending_archiveresults, thread_sensitive=True)(snapshot)
                     await sync_to_async(snapshot.create_pending_archiveresults, thread_sensitive=True)(hooks=hooks)
                 try:
-                    await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
+                    await sync_to_async(snapshot.advance_lifecycle, thread_sensitive=True)()
                 except ValidationError as err:
                     if "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs." not in str(err):
                         raise

archivebox/services/tag_service.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from asgiref.sync import sync_to_async
+
 from abx_dl.events import TagEvent
 from abx_dl.services.base import BaseService
 
@@ -13,10 +15,10 @@ class TagService(BaseService):
         self.bus.on(TagEvent, self.on_TagEvent__save_to_db)
 
     async def on_TagEvent__save_to_db(self, event: TagEvent) -> None:
-        from archivebox.core.models import Snapshot, SnapshotTag, Tag
+        from archivebox.core.models import Snapshot, Tag
 
         snapshot = await Snapshot.objects.filter(id=event.snapshot_id).afirst()
         if snapshot is None:
             return
-        tag, _ = await Tag.objects.aget_or_create(name=event.name)
-        await SnapshotTag.objects.aget_or_create(snapshot=snapshot, tag=tag)
+        tag, _ = await sync_to_async(Tag.get_or_create_by_name, thread_sensitive=True)(event.name)
+        await sync_to_async(snapshot.add_tag_ids, thread_sensitive=True)([tag.pk])

archivebox/tests/conftest.py

@@ -1660,6 +1660,74 @@ def resolve_abxpkg_binary_env(
     return {str(key): str(value) for key, value in payload.items()}
 
 
+def run_test_hook(
+    script: Path,
+    output_dir: Path,
+    config: dict[str, Any] | None = None,
+    timeout: int = 60,
+    **arguments: Any,
+):
+    """Execute a shipped finite hook through abx-dl and ArchiveBox's real DB projector."""
+    import asyncio
+
+    from abx_dl.execution import execute_hook
+    from abx_dl.orchestrator import create_bus
+    from archivebox.machine.models import Process
+    from archivebox.plugins.discovery import get_plugin_catalog
+    from archivebox.services.process_service import ProcessService, parse_event_datetime
+
+    resolved_script = script.resolve()
+    hook = next(
+        (hook for plugin in get_plugin_catalog().values() for hook in plugin.hooks if hook.path.resolve() == resolved_script),
+        None,
+    )
+    assert hook is not None, f"shipped hook is not in the plugin catalog: {script}"
+    assert not hook.is_background, f"run_test_hook only supports finite hooks: {hook.full_name}"
+
+    env = os.environ.copy()
+    for key, value in (config or {}).items():
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            env[key] = "true" if value else "false"
+        elif isinstance(value, (dict, list, tuple)):
+            env[key] = json.dumps(value)
+        else:
+            env[key] = str(value)
+    if env.get("NODE_MODULES_DIR"):
+        env.setdefault("NODE_MODULE_DIR", env["NODE_MODULES_DIR"])
+
+    bus = create_bus(name=f"test_hook_{hook.plugin_name}", total_timeout=float(timeout) + 30.0)
+    ProcessService(bus)
+
+    async def execute_and_close():
+        try:
+            return await execute_hook(
+                hook,
+                output_dir=output_dir,
+                env=env,
+                arguments=arguments,
+                timeout=timeout,
+                bus=bus,
+                process_type=Process.TypeChoices.HOOK,
+            )
+        finally:
+            await bus.wait_until_idle()
+            await bus.destroy(clear=False)
+
+    completed = asyncio.run(execute_and_close())
+    process = (
+        Process.objects.filter(
+            pid=completed.pid or None,
+            started_at=parse_event_datetime(completed.start_ts),
+        )
+        .order_by("-modified_at")
+        .first()
+    )
+    assert process is not None, f"hook completed without an ArchiveBox Process projection: {hook.full_name}"
+    return process
+
+
 def resolve_abxpkg_chrome_env(lib_dir: Path, env: dict[str, str] | None = None) -> dict[str, str]:
     from abx_plugins import get_plugins_dir
 
@@ -1683,7 +1751,7 @@ def install_real_binary(
     binproviders: str = "env",
     overrides: dict[str, dict[str, Any]] | None = None,
 ):
-    """Install and persist a real binary through the normal Binary state machine."""
+    """Install and persist a real binary through the normal Binary lifecycle."""
     from archivebox.machine.models import Binary, Machine
 
     binary = Binary.objects.create(
@@ -1693,7 +1761,7 @@ def install_real_binary(
         overrides=overrides or {},
         status=Binary.StatusChoices.QUEUED,
     )
-    assert binary.tick_claimed(lock_seconds=600)
+    assert binary.install_claimed(lock_seconds=600)
     binary.refresh_from_db()
     assert binary.status == Binary.StatusChoices.INSTALLED
     assert binary.retry_at is None

archivebox/tests/test_api_v1_core_archiveresults.py

@@ -3,6 +3,7 @@ from datetime import timedelta
 import pytest
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import connection
+from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart
 from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 
@@ -14,7 +15,7 @@ from archivebox.tests.conftest import api_client_request
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_archiveresult_upload_upserts_one_snapshot_plugin_result(client, api_admin_user, api_headers):
+def test_archiveresult_upload_upserts_by_snapshot_plugin_and_hook(client, api_admin_user, api_headers):
     crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
     snapshot = Snapshot.objects.create(url="https://example.com/unified-result", crawl=crawl)
 
@@ -39,13 +40,130 @@ def test_archiveresult_upload_upserts_one_snapshot_plugin_result(client, api_adm
     assert server_response.status_code == 200, server_response.content
     assert extension_update.status_code == 200, extension_update.content
     assert extension_update.json()["id"] == extension_response.json()["id"]
-    assert server_response.json()["id"] == extension_response.json()["id"]
+    assert server_response.json()["id"] != extension_response.json()["id"]
 
-    results = ArchiveResult.objects.filter(snapshot=snapshot, plugin="screenshot")
-    assert results.count() == 1
-    result = results.get()
-    assert result.hook_name == "on_Snapshot__archivebox_browser_extension_upload"
-    assert set(result.output_files) == {"browser.png", "server.png", "browser-2.png"}
+    results = ArchiveResult.objects.filter(snapshot=snapshot, plugin="screenshot").order_by("hook_name")
+    assert results.count() == 2
+    extension_result = results.get(hook_name="on_Snapshot__archivebox_browser_extension_upload")
+    server_result = results.get(hook_name="on_Snapshot__50_screenshot")
+    assert set(extension_result.output_files) == {"browser.png", "browser-2.png"}
+    assert set(server_result.output_files) == {"server.png"}
+    snapshot.refresh_from_db()
+    assert snapshot.output_size == len(b"browser") + len(b"browser-2") + len(b"server")
+
+
+def test_archiveresult_create_does_not_open_a_database_transaction(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/autocommit-result", crawl=crawl)
+
+    with CaptureQueriesContext(connection) as queries:
+        response = client.post(
+            "/api/v1/core/archiveresults",
+            {
+                "snapshot_id": str(snapshot.id),
+                "plugin": "chrome_extension_screenshot",
+                "files": SimpleUploadedFile("screenshot.png", b"screenshot", content_type="image/png"),
+                "output_paths": "screenshot.png",
+            },
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_intermediate_archiveresult_chunks_only_write_to_disk(client, api_admin_user, api_headers):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/chunked-result", crawl=crawl)
+    create_response = client.post(
+        "/api/v1/core/archiveresults",
+        {
+            "snapshot_id": str(snapshot.id),
+            "plugin": "chrome_extension_mhtml",
+            "status": ArchiveResult.StatusChoices.STARTED,
+        },
+        **api_headers,
+    )
+    assert create_response.status_code == 200, create_response.content
+    result = ArchiveResult.objects.get(pk=create_response.json()["id"])
+    original_modified_at = result.modified_at
+
+    with CaptureQueriesContext(connection) as queries:
+        chunk_response = client.patch(
+            f"/api/v1/core/archiveresult/{result.id}",
+            encode_multipart(
+                BOUNDARY,
+                {
+                    "files": SimpleUploadedFile("snapshot.mhtml.part-000000", b"first", content_type="multipart/related"),
+                    "chunk_output_path": "snapshot.mhtml",
+                    "chunk_index": "0",
+                    "chunk_count": "2",
+                    "chunk_offset": "0",
+                    "chunk_total_size": "11",
+                    "mime_type": "multipart/related",
+                    "status": ArchiveResult.StatusChoices.STARTED,
+                },
+            ),
+            content_type=MULTIPART_CONTENT,
+            **api_headers,
+        )
+
+    assert chunk_response.status_code == 200, chunk_response.content
+    result.refresh_from_db()
+    assert result.output_files == {}
+    assert result.output_size == 0
+    assert result.modified_at == original_modified_at
+    writes = [query["sql"] for query in queries if query["sql"].lstrip().upper().startswith(("INSERT", "UPDATE", "DELETE"))]
+    assert writes == []
+
+    final_response = client.patch(
+        f"/api/v1/core/archiveresult/{result.id}",
+        encode_multipart(
+            BOUNDARY,
+            {
+                "files": SimpleUploadedFile("snapshot.mhtml.part-000001", b"-final", content_type="multipart/related"),
+                "chunk_output_path": "snapshot.mhtml",
+                "chunk_index": "1",
+                "chunk_count": "2",
+                "chunk_offset": "5",
+                "chunk_total_size": "11",
+                "mime_type": "multipart/related",
+                "status": ArchiveResult.StatusChoices.SUCCEEDED,
+            },
+        ),
+        content_type=MULTIPART_CONTENT,
+        **api_headers,
+    )
+    assert final_response.status_code == 200, final_response.content
+    result.refresh_from_db()
+    assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
+    assert result.output_size == 11
+    assert result.output_files["snapshot.mhtml"]["upload"]["complete"] is True
+    snapshot.refresh_from_db()
+    assert snapshot.output_size == 11
+
+
+def test_archiveresult_safe_update_rejects_stale_writers(api_admin_user):
+    crawl = Crawl.objects.create(urls="https://example.com", created_by=api_admin_user)
+    snapshot = Snapshot.objects.create(url="https://example.com/cas-result", crawl=crawl)
+    result = ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="chrome_extension_dom",
+        hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
+        status=ArchiveResult.StatusChoices.STARTED,
+        output_size=1,
+    )
+    stale_result = ArchiveResult.objects.get(pk=result.pk)
+
+    assert result.safe_update({"output_size": 2}) is True
+    assert stale_result.safe_update({"output_size": 3}) is False
+
+    stale_result.refresh_from_db()
+    snapshot.refresh_from_db()
+    assert stale_result.output_size == 2
+    assert snapshot.output_size == 2
 
 
 def test_archiveresult_upload_api_queues_snapshot_maintenance_without_finalizing(client, api_admin_user, api_headers):

archivebox/tests/test_api_v1_core_snapshots.py

@@ -2,6 +2,8 @@ import threading
 import time
 
 import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot
 from archivebox.crawls.locks import crawl_lifecycle_lock
@@ -85,3 +87,66 @@ def test_existing_snapshot_metadata_sync_does_not_wait_for_active_crawl(client,
     assert response.status_code == 200, response.content
     assert response.json()["id"] == str(snapshot.id)
     assert elapsed < 1
+
+
+def test_new_snapshot_creation_does_not_open_a_database_transaction(client, api_admin_user, api_headers):
+    url = "https://example.com/browser-extension-new-snapshot"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+
+    with CaptureQueriesContext(connection) as queries:
+        response = client.post(
+            "/api/v1/core/snapshots",
+            data={
+                "url": url,
+                "crawl_id": str(crawl.id),
+                "depth": 0,
+                "status": Snapshot.StatusChoices.QUEUED,
+                "tags": ["browser-extension-upload"],
+            },
+            content_type="application/json",
+            **api_headers,
+        )
+
+    assert response.status_code == 200, response.content
+    assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
+    assert Snapshot.objects.get(url=url, crawl=crawl).tags.filter(name="browser-extension-upload").exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_new_snapshot_creation_does_not_wait_for_active_crawl(client, api_admin_user, api_headers):
+    url = "https://example.com/browser-extension-new-snapshot-active-crawl"
+    crawl = Crawl.objects.create(urls=url, created_by=api_admin_user)
+    lock_acquired = threading.Event()
+    release_lock = threading.Event()
+
+    def hold_active_crawl_lock():
+        with crawl_lifecycle_lock(str(crawl.id)):
+            lock_acquired.set()
+            release_lock.wait(timeout=3)
+
+    holder = threading.Thread(target=hold_active_crawl_lock)
+    holder.start()
+    assert lock_acquired.wait(timeout=1)
+
+    started_at = time.monotonic()
+    response = client.post(
+        "/api/v1/core/snapshots",
+        data={
+            "url": url,
+            "crawl_id": str(crawl.id),
+            "depth": 0,
+            "status": Snapshot.StatusChoices.QUEUED,
+            "tags": ["browser-extension-upload"],
+        },
+        content_type="application/json",
+        **api_headers,
+    )
+    elapsed = time.monotonic() - started_at
+    release_lock.set()
+    holder.join(timeout=3)
+
+    assert response.status_code == 200, response.content
+    assert Snapshot.objects.filter(url=url, crawl=crawl).count() == 1
+    assert elapsed < 1

archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py

@@ -1,4 +1,6 @@
 import pytest
+from django.db import IntegrityError, connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot, Tag
 from archivebox.crawls.models import Crawl
@@ -13,13 +15,39 @@ def test_basic_success_case_request(client, api_admin_user, api_headers):
     snapshot = Snapshot.objects.create(url="https://example.com/tag-add", crawl=crawl)
     tag = Tag.objects.create(name="api-basic-add-tag", created_by=api_admin_user)
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/tags/add-to-snapshot/",
-        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
-        headers=api_headers,
-    )
+    with CaptureQueriesContext(connection) as queries:
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/tags/add-to-snapshot/",
+            payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+            headers=api_headers,
+        )
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+    assert snapshot.tags.filter(pk=tag.pk).exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []
+
+
+def test_add_tag_ids_reraises_non_duplicate_integrity_errors(snapshot):
+    with pytest.raises(IntegrityError):
+        snapshot.add_tag_ids([2**31 - 1])
+
+
+def test_add_tag_ids_treats_existing_snapshot_tag_as_idempotent(snapshot, admin_user):
+    tag = Tag.objects.create(name="already-attached", created_by=admin_user)
+
+    snapshot.add_tag_ids([tag.pk])
+    snapshot.add_tag_ids([tag.pk])
+
+    assert snapshot.tags.filter(pk=tag.pk).count() == 1
+
+
+def test_save_tags_without_creator_uses_tag_model_default(snapshot):
+    snapshot.save_tags(["model-default-creator"])
+
+    tag = Tag.objects.get(name="model-default-creator")
+    assert tag.created_by_id is not None

archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py

@@ -1,4 +1,6 @@
 import pytest
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 
 from archivebox.core.models import Snapshot, Tag
 from archivebox.crawls.models import Crawl
@@ -14,13 +16,18 @@ def test_basic_success_case_request(client, api_admin_user, api_headers):
     tag = Tag.objects.create(name="api-basic-remove-tag", created_by=api_admin_user)
     snapshot.tags.add(tag)
 
-    response = api_client_request(
-        client,
-        "post",
-        "/api/v1/core/tags/remove-from-snapshot/",
-        payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
-        headers=api_headers,
-    )
+    with CaptureQueriesContext(connection) as queries:
+        response = api_client_request(
+            client,
+            "post",
+            "/api/v1/core/tags/remove-from-snapshot/",
+            payload={"snapshot_id": str(snapshot.id), "tag_id": tag.id},
+            headers=api_headers,
+        )
 
     assert response.status_code == 200, response.content
     assert response.json()["success"] is True
+    assert not snapshot.tags.filter(pk=tag.pk).exists()
+    if connection.vendor == "sqlite":
+        transaction_queries = [query["sql"] for query in queries if query["sql"].strip().upper() in {"BEGIN", "COMMIT"}]
+        assert transaction_queries == []

archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py

@@ -195,7 +195,7 @@ def test_crawl_pause_resume_api_cascades_archiveresults_and_leaves_finished_snap
             hook_name="on_Snapshot__93_hashes.py",
             lib_dir=lib_dir,
         )
-        sealed_snapshot.sm.seal()
+        sealed_snapshot.seal()
         sealed_snapshot.refresh_from_db()
         assert sealed_snapshot.status == Snapshot.StatusChoices.SEALED
         assert sealed_snapshot.retry_at is None

archivebox/tests/test_archive_result_service.py

@@ -19,9 +19,9 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 
 def _snapshot_hook_name(plugin_name: str) -> str:
-    from abx_dl.models import discover_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
-    plugin = discover_plugins().get(plugin_name)
+    plugin = get_plugin_catalog().get(plugin_name)
     assert plugin is not None, f"missing test plugin {plugin_name}"
     hooks = plugin.filter_hooks("Snapshot")
     assert hooks, f"missing Snapshot hooks for {plugin_name}"
@@ -47,15 +47,15 @@ def _run_shipped_snapshot_hook(
     """Run one shipped hook through the production process/result bus services."""
     import asyncio
 
-    from abx_dl.models import discover_plugins
     from abx_dl.services.process_service import ProcessService as HookProcessService
     from abx_plugins.plugins.base.utils import get_hydrated_required_binaries
     from archivebox.core.models import ArchiveResult
     from archivebox.machine.models import Process
+    from archivebox.plugins.discovery import get_plugin_catalog
     from archivebox.services.archive_result_service import ArchiveResultService
     from archivebox.services.process_service import ProcessService as PersistedProcessService
 
-    discovered_plugin = discover_plugins().get(plugin)
+    discovered_plugin = get_plugin_catalog().get(plugin)
     assert discovered_plugin is not None, f"missing test plugin {plugin}"
     matching_hooks = [hook for hook in discovered_plugin.filter_hooks("Snapshot") if hook.name == hook_name or hook.path.name == hook_name]
     assert len(matching_hooks) == 1, f"missing or ambiguous Snapshot hook {plugin}:{hook_name}"
@@ -230,7 +230,7 @@ def test_archiveresult_event_retry_updates_existing_hook_row(tmp_path, hermetic_
     _cleanup_machine_process_rows()
 
 
-def test_archiveresult_duplicate_plugin_rows_are_rejected():
+def test_archiveresult_duplicate_hook_rows_are_rejected():
     from django.db import IntegrityError, transaction
     from archivebox.core.models import ArchiveResult
 
@@ -246,45 +246,35 @@ def test_archiveresult_duplicate_plugin_rows_are_rejected():
         ArchiveResult.objects.create(
             snapshot=snapshot,
             plugin="wget",
-            hook_name="on_Snapshot__99_other_wget_hook",
+            hook_name="on_Snapshot__06_wget.finite.bg",
             status=ArchiveResult.StatusChoices.SUCCEEDED,
         )
 
 
-def test_archivewebpage_lifecycle_hooks_project_one_plugin_output():
+def test_archiveresult_event_create_uses_one_result_lookup():
     from abx_dl.events import ArchiveResultEvent
-    from archivebox.core.models import ArchiveResult
+    from django.db import connection
+    from django.test.utils import CaptureQueriesContext
+
     from archivebox.services.archive_result_service import _save_archiveresult_event_to_db
 
     snapshot = _create_snapshot()
-    _save_archiveresult_event_to_db(
-        ArchiveResultEvent(
-            snapshot_id=str(snapshot.id),
-            plugin="archivewebpage",
-            hook_name="on_Snapshot__16_archivewebpage_start",
-            status="succeeded",
-            output_str="recording started",
-            output_files=[OutputFile(path="recording.json", extension="json", mimetype="application/json", size=175)],
-        ),
-        None,
-    )
-    _save_archiveresult_event_to_db(
-        ArchiveResultEvent(
-            snapshot_id=str(snapshot.id),
-            plugin="archivewebpage",
-            hook_name="on_Snapshot__65_archivewebpage_stop",
-            status="succeeded",
-            output_str="archivewebpage.wacz",
-            output_files=[OutputFile(path="archivewebpage.wacz", extension="wacz", size=2048)],
-        ),
-        None,
+    event = ArchiveResultEvent(
+        snapshot_id=str(snapshot.id),
+        plugin="review-query-count",
+        hook_name="on_Snapshot__99_review.py",
+        status="failed",
     )
 
-    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="archivewebpage")
-    assert result.hook_name == "on_Snapshot__65_archivewebpage_stop"
-    assert result.output_str == "archivewebpage.wacz"
-    assert set(result.output_files) == {"recording.json", "archivewebpage.wacz"}
-    assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="archivewebpage").count() == 1
+    with CaptureQueriesContext(connection) as queries:
+        _save_archiveresult_event_to_db(event, None)
+
+    result_lookups = [
+        query["sql"]
+        for query in queries
+        if query["sql"].lstrip().upper().startswith("SELECT") and 'FROM "core_archiveresult"' in query["sql"]
+    ]
+    assert len(result_lookups) == 1
 
 
 def test_process_completed_projects_failed_archiveresult_from_shipped_hook(tmp_path, hermetic_lib_dir):
@@ -470,13 +460,12 @@ def test_retry_failed_archiveresults_requeues_snapshot_in_queued_state():
     reset_count = snapshot.retry_failed_archiveresults()
 
     snapshot.refresh_from_db()
-    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="chrome")
+    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="chrome", hook_name="on_Snapshot__11_chrome_wait")
     assert reset_count == 1
     assert snapshot.status == Snapshot.StatusChoices.QUEUED
     assert snapshot.retry_at is not None
     assert snapshot.current_step == 0
     assert result.status == ArchiveResult.StatusChoices.QUEUED
-    assert result.hook_name == ""
     assert result.output_str == ""
     assert result.output_json is None
     assert result.output_files == {}
@@ -554,14 +543,17 @@ def test_collect_output_metadata_preserves_file_metadata():
 
 
 def test_collect_output_metadata_detects_warc_gz_mimetype(tmp_path):
-    from archivebox.services.archive_result_service import _collect_output_metadata
+    from abx_dl.output_files import OutputManifest
 
     plugin_dir = tmp_path / "wget"
     warc_file = plugin_dir / "warc" / "capture.warc.gz"
     warc_file.parent.mkdir(parents=True, exist_ok=True)
     warc_file.write_bytes(b"warc-bytes")
 
-    output_files, output_size, output_mimetypes = _collect_output_metadata(plugin_dir)
+    manifest = OutputManifest.scan(plugin_dir)
+    output_files = manifest.as_mapping()
+    output_size = manifest.total_size
+    output_mimetypes = ",".join(manifest.mimetypes)
 
     assert output_files["warc/capture.warc.gz"] == {
         "extension": "gz",

archivebox/tests/test_binary_service.py

@@ -22,13 +22,13 @@ def _runtime_env(data_dir: Path, *, lib_dir: Path | None = None, **extra: str) -
     }
 
 
-def _run_real_binary_state_machine(data_dir: Path, *, name: str, binproviders: str, env: dict[str, str]):
-    """Run a real Binary model through its abxpkg-backed state machine."""
+def _run_real_binary_lifecycle(data_dir: Path, *, name: str, binproviders: str, env: dict[str, str]):
+    """Run a real Binary model through its abxpkg-backed lifecycle."""
     script = (
         "from archivebox.machine.models import Binary, Machine; "
         f"binary = Binary.objects.create(machine=Machine.current(), name={name!r}, binproviders={binproviders!r}, status=Binary.StatusChoices.QUEUED); "
-        "assert binary.tick_claimed(lock_seconds=600); "
-        "print('BINARY_STATE_MACHINE_E2E_DONE')"
+        "assert binary.install_claimed(lock_seconds=600); "
+        "print('BINARY_LIFECYCLE_E2E_DONE')"
     )
     return run_archivebox_cmd(
         ["shell", "-c", script],
@@ -59,7 +59,7 @@ def test_binary_request_preserves_native_overrides_in_db():
         status=Binary.StatusChoices.QUEUED,
         retry_at=timezone.now(),
     )
-    assert binary.tick_claimed(lock_seconds=600)
+    assert binary.install_claimed(lock_seconds=600)
     binary.refresh_from_db()
     assert binary.status == Binary.StatusChoices.INSTALLED
     assert Path(binary.abspath).resolve() == Path(sys.executable).resolve()
@@ -98,11 +98,11 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
     host_binary = shutil.which(name)
     assert host_binary is not None
     runtime_env = _runtime_env(initialized_archive)
-    _cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env)
+    _cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert returncode == 0, stderr
-    assert "BINARY_STATE_MACHINE_E2E_DONE" in stdout
+    assert "BINARY_LIFECYCLE_E2E_DONE" in stdout
 
     with use_archivebox_db(initialized_archive):
         binary = Binary.objects.get(name=name)
@@ -203,7 +203,7 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
     provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
     runtime_env = _runtime_env(initialized_archive)
 
-    _cmd_result = _run_real_binary_state_machine(initialized_archive, name=name, binproviders="env", env=runtime_env)
+    _cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
     stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
     assert returncode != 0, stdout + stderr

archivebox/tests/test_cli_archiveresult.py

@@ -61,11 +61,10 @@ class TestArchiveResultCreate:
 
         ar = next(r for r in records if r["type"] == "ArchiveResult")
         assert ar["plugin"] == "title"
-        # Queue projection is one row per plugin, while a plugin can contain
-        # several ordered hooks. The runner records the concrete hook only
-        # after execution; inventing one here would make the pending row claim
-        # work that has not run and reintroduce hook-level duplicate results.
-        assert ar["hook_name"] == ""
+        # Hook identity is known at scheduling time and is the durable retry
+        # key. Emitting it here prevents multiple hooks in one plugin from
+        # collapsing into the same ArchiveResult row.
+        assert ar["hook_name"] == "on_Snapshot__54_title"
         assert "id" not in ar
 
     def test_create_with_specific_plugin(self, initialized_archive):
@@ -92,11 +91,9 @@ class TestArchiveResultCreate:
         assert code == 0
         records = parse_jsonl_output(stdout2)
         ar_records = [r for r in records if r.get("type") == "ArchiveResult"]
-        assert len(ar_records) >= 1
+        assert len(ar_records) == 1
         assert all(record["plugin"] == "screenshot" for record in ar_records)
-        # A requested plugin is the schedulable unit; its concrete hook is an
-        # execution result, not input metadata on this pre-execution request.
-        assert all(record["hook_name"] == "" for record in ar_records)
+        assert [record["hook_name"] for record in ar_records] == ["on_Snapshot__51_screenshot"]
 
     def test_create_pass_through_crawl(self, initialized_archive):
         """Pass-through Crawl records unchanged."""

archivebox/tests/test_cli_extract_input.py

@@ -22,7 +22,7 @@ def create_extract_snapshot(initialized_archive, env, url="https://example.com")
     )
 
 
-def test_extract_archiveresult_record_queues_one_plugin_result(initialized_archive):
+def test_extract_archiveresult_record_queues_only_exact_hook(initialized_archive):
     env = cli_env(PLUGINS="archivewebpage")
     create_extract_snapshot(initialized_archive, env)
 
@@ -52,7 +52,7 @@ def test_extract_archiveresult_record_queues_one_plugin_result(initialized_archi
         rows = list(
             ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="archivewebpage").values_list("hook_name", "status"),
         )
-    assert rows == [("", ArchiveResult.StatusChoices.QUEUED)]
+    assert rows == [(hook_name, ArchiveResult.StatusChoices.QUEUED)]
 
 
 def test_extract_runs_on_snapshot_id(initialized_archive):

archivebox/tests/test_cli_piping.py

@@ -38,7 +38,7 @@ PIPE_TEST_ENV = {
 
 def run_real_txt_parser(tmp_path, text):
     """Run the shipped text parser and return its real snapshot output directory."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "parser-snapshot"
     staticfile_dir = snap_dir / "staticfile"
@@ -47,7 +47,7 @@ def run_real_txt_parser(tmp_path, text):
     output_dir.mkdir(parents=True)
     (staticfile_dir / "input.txt").write_text(text, encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_cli_run.py

@@ -240,7 +240,7 @@ class TestRunWithArchiveResult:
     """Tests for `archivebox run` with ArchiveResult input."""
 
     @pytest.mark.django_db(transaction=True)
-    def test_run_treats_no_id_archiveresult_request_as_plugin_work(self, initialized_archive):
+    def test_run_creates_and_runs_exact_no_id_archiveresult_request(self, initialized_archive):
         import json
 
         from archivebox.core.models import ArchiveResult
@@ -284,9 +284,13 @@ class TestRunWithArchiveResult:
                     "output_str",
                 ),
             )
-        assert len(rows) == 1
-        assert rows[0][0] != missing_hook
-        assert rows[0][1] in ArchiveResult.FINAL_STATES
+        assert rows == [
+            (
+                missing_hook,
+                ArchiveResult.StatusChoices.FAILED,
+                "Queued hook is no longer available in the installed plugin",
+            ),
+        ]
 
     def test_run_requeues_failed_archiveresult(self, initialized_archive):
         """Run re-queues a failed ArchiveResult."""
@@ -1072,7 +1076,7 @@ class TestRecoverOrchestratorState:
         assert snapshot.retry_at is None
         assert snapshot.downloaded_at is not None
 
-    def test_create_pending_archiveresults_creates_one_plugin_row_not_hook_rows(self):
+    def test_create_pending_archiveresults_uses_canonical_hook_names(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -1092,17 +1096,11 @@ class TestRecoverOrchestratorState:
             retry_at=timezone.now(),
         )
 
-        results = snapshot.create_pending_archiveresults(
-            hooks=[
-                ("chrome", "on_Snapshot__00_chrome_launch.daemon.bg"),
-                ("chrome", "on_Snapshot__01_chrome_tab.daemon.bg"),
-                ("chrome", "on_Snapshot__30_chrome_navigate"),
-                ("title", "on_Snapshot__54_title"),
-            ],
-        )
+        snapshot.create_pending_archiveresults()
 
-        assert [(result.plugin, result.hook_name) for result in results] == [("chrome", ""), ("title", "")]
-        assert ArchiveResult.objects.filter(snapshot=snapshot).count() == 2
+        hook_names = list(ArchiveResult.objects.filter(snapshot=snapshot).values_list("hook_name", flat=True))
+        assert hook_names
+        assert all(not hook_name.endswith((".py", ".js", ".sh")) for hook_name in hook_names)
 
     def test_snapshot_hooks_for_pending_archiveresults_respects_disabled_plugins_when_plugins_empty(self):
         from django.utils import timezone
@@ -1697,13 +1695,11 @@ class TestRecoverOrchestratorState:
         Snapshot.objects.filter(pk=snapshot.pk).update(fs_version="0.9.0")
         snapshot.refresh_from_db()
         snapshot.output_dir.mkdir(parents=True, exist_ok=True)
-        title_dir = snapshot.output_dir / "title"
-        title_dir.mkdir(parents=True, exist_ok=True)
-        (title_dir / "title.txt").write_text("Example Domain\n", encoding="utf-8")
+        (snapshot.output_dir / "source.txt").write_text("real targeted maintenance input\n", encoding="utf-8")
         result = ArchiveResult.objects.create(
             snapshot=snapshot,
-            plugin="search_backend_sqlite",
-            hook_name="on_Snapshot__90_index_sqlite",
+            plugin="hashes",
+            hook_name="on_Snapshot__93_hashes.py",
             status=ArchiveResult.StatusChoices.QUEUED,
         )
 
@@ -1718,7 +1714,7 @@ class TestRecoverOrchestratorState:
         assert result.end_ts is not None
 
     @pytest.mark.django_db(transaction=True)
-    def test_run_due_snapshot_runs_plugin_for_obsolete_queued_hook_name(self):
+    def test_run_due_snapshot_fails_obsolete_queued_hook_name(self):
         from django.utils import timezone
 
         from archivebox.base_models.models import get_or_create_system_user_pk
@@ -1749,8 +1745,8 @@ class TestRecoverOrchestratorState:
 
         result.refresh_from_db()
         snapshot.refresh_from_db()
-        assert result.status in ArchiveResult.FINAL_STATES
-        assert result.hook_name != "on_Snapshot__50_singlefile.py"
+        assert result.status == ArchiveResult.StatusChoices.FAILED
+        assert result.output_str == "Queued hook is no longer available in the installed plugin"
         assert snapshot.retry_at is None
 
     @pytest.mark.django_db(transaction=True)
@@ -1829,10 +1825,7 @@ class TestRecoverOrchestratorState:
         recursive_test_site,
         chrome_isolation,
     ):
-        from pathlib import Path
-
-        from archivebox.core.models import ArchiveResult, Snapshot
-        from archivebox.machine.models import Process
+        from archivebox.core.models import ArchiveResult
         from archivebox.tests.test_orm_helpers import use_archivebox_db
 
         env = cli_env(disable_extractors=True)
@@ -1870,19 +1863,13 @@ class TestRecoverOrchestratorState:
         snapshot_id = navigate_record["snapshot_id"]
 
         with use_archivebox_db(initialized_archive):
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            tab_processes = [
-                process
-                for process in Process.objects.filter(pwd=str(Path(snapshot.output_dir) / "chrome")).order_by("-started_at")
-                if Path(process.hook_script_name or "").stem == "on_Snapshot__01_chrome_tab.daemon.bg"
-            ]
-            assert tab_processes
-            first_tab_process_id = tab_processes[0].id
-            plugin_result = ArchiveResult.objects.get(
+            tab_result = ArchiveResult.objects.get(
                 snapshot_id=snapshot_id,
                 plugin="chrome",
+                hook_name="on_Snapshot__01_chrome_tab.daemon.bg",
             )
-            assert plugin_result.hook_name == "on_Snapshot__30_chrome_navigate"
+            first_tab_process_id = tab_result.process_id
+            assert first_tab_process_id is not None
 
         update_process = run_archivebox_cmd(
             ["archiveresult", "update", "--status=queued"],
@@ -1916,18 +1903,18 @@ class TestRecoverOrchestratorState:
             navigate_result = ArchiveResult.objects.get(
                 snapshot_id=snapshot_id,
                 plugin="chrome",
+                hook_name="on_Snapshot__30_chrome_navigate",
+            )
+            tab_result = ArchiveResult.objects.get(
+                snapshot_id=snapshot_id,
+                plugin="chrome",
+                hook_name="on_Snapshot__01_chrome_tab.daemon.bg",
             )
-            snapshot = Snapshot.objects.get(id=snapshot_id)
-            tab_processes = [
-                process
-                for process in Process.objects.filter(pwd=str(Path(snapshot.output_dir) / "chrome")).order_by("-started_at")
-                if Path(process.hook_script_name or "").stem == "on_Snapshot__01_chrome_tab.daemon.bg"
-            ]
 
         assert run_process.returncode == 0
         assert navigate_result.status == ArchiveResult.StatusChoices.SUCCEEDED
-        assert navigate_result.hook_name == "on_Snapshot__30_chrome_navigate"
-        assert tab_processes[0].id != first_tab_process_id
+        assert tab_result.process_id is not None
+        assert tab_result.process_id != first_tab_process_id
 
     def test_recover_orchestrator_state_ignores_sealed_downloaded_snapshot_without_results(self):
         from django.utils import timezone
@@ -2119,7 +2106,7 @@ class TestRunDueCrawlState:
             retry_at=timezone.now(),
         )
 
-        snapshot.sm.tick()
+        snapshot.advance_lifecycle()
         snapshot.refresh_from_db()
 
         assert snapshot.status == Snapshot.StatusChoices.STARTED
@@ -2186,6 +2173,7 @@ class TestRunDueCrawlState:
         assert finished.output_str == "keep me"
         assert finished.output_files == {"favicon.ico": {"size": 1}}
 
+    @pytest.mark.django_db(transaction=True)
     def test_finished_parser_result_projects_children_before_resume_seals_snapshot(self):
         from importlib.resources import files
         from pathlib import Path
@@ -2195,7 +2183,7 @@ class TestRunDueCrawlState:
         from archivebox.base_models.models import get_or_create_system_user_pk
         from archivebox.core.models import ArchiveResult, Snapshot
         from archivebox.crawls.models import Crawl
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
+        from archivebox.tests.conftest import run_test_hook
         from archivebox.services.runner import run_due_snapshot
 
         crawl = Crawl.objects.create(
@@ -2221,7 +2209,7 @@ class TestRunDueCrawlState:
             encoding="utf-8",
         )
         hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             parser_dir,
             config={"ABXPKG_LIB_DIR": str(root.output_dir.parent.parent / "lib"), "SNAP_DIR": str(root.output_dir)},
@@ -2232,12 +2220,12 @@ class TestRunDueCrawlState:
         )
         process.refresh_from_db()
         assert process.exit_code == 0, process.stderr
-        result_record = next(record for record in extract_records_from_process(process) if record.get("type") == "ArchiveResult")
+        result_record = next(record for record in process.get_records() if record.get("type") == "ArchiveResult")
         ArchiveResult.objects.create(
             snapshot=root,
             process=process,
-            plugin=result_record["plugin"],
-            hook_name=result_record["hook_name"],
+            plugin="parse_txt_urls",
+            hook_name=hook_path.name,
             status=result_record["status"],
             output_str=result_record.get("output_str", ""),
             output_files={"urls.jsonl": {"size": (parser_dir / "urls.jsonl").stat().st_size}},
@@ -3005,7 +2993,7 @@ class TestRecoverOrchestratorStateRedFailureModes:
             modified_at=now + timedelta(seconds=1),
         )
 
-        snapshot.sm.seal()
+        snapshot.seal()
         snapshot.refresh_from_db()
         assert snapshot.status == Snapshot.StatusChoices.SEALED
         assert snapshot.retry_at is None

archivebox/tests/test_cli_update_reindex_snapshots.py

@@ -15,7 +15,7 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
-def test_only_sealed_search_backfill_bypasses_snapshot_lifecycle():
+def test_targeted_plugin_retries_preserve_sealed_snapshot_lifecycle():
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.cli.archivebox_extract import run_plugins
     from archivebox.core.models import ArchiveResult
@@ -85,8 +85,8 @@ def test_only_sealed_search_backfill_bypasses_snapshot_lifecycle():
     )
     extract_crawl.refresh_from_db()
     extract_snapshot.refresh_from_db()
-    assert extract_crawl.status == Crawl.StatusChoices.QUEUED
-    assert extract_snapshot.status == Snapshot.StatusChoices.QUEUED
+    assert extract_crawl.status == Crawl.StatusChoices.SEALED
+    assert extract_snapshot.status == Snapshot.StatusChoices.SEALED
     assert extract_snapshot.archiveresult_set.filter(
         plugin="wget",
         status=ArchiveResult.StatusChoices.QUEUED,

archivebox/tests/test_config_MAX_limits.py

@@ -67,7 +67,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import CrawlRunner
 
     crawl = Crawl.objects.create(
@@ -92,7 +92,7 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
         encoding="utf-8",
     )
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         parser_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_crawl_runner.py

@@ -261,7 +261,9 @@ def test_snapshot_started_state_keeps_retry_at_lease():
         retry_at=before,
     )
 
-    assert snapshot.tick_claimed(lock_seconds=60) is True
+    assert snapshot.claim_processing_lock(lock_seconds=60) is True
+    snapshot.refresh_from_db()
+    assert snapshot.advance_lifecycle() is True
 
     snapshot.refresh_from_db()
     assert snapshot.status == Snapshot.StatusChoices.STARTED
@@ -269,6 +271,45 @@ def test_snapshot_started_state_keeps_retry_at_lease():
     assert snapshot.retry_at > before
 
 
+@pytest.mark.django_db(transaction=True)
+def test_system_update_crawl_runs_database_maintenance_without_snapshot_work():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.services.runner import CrawlRunner
+    from django.utils import timezone
+
+    owner_id = get_or_create_system_user_pk()
+    archived_crawl = Crawl.objects.create(
+        urls="https://example.com",
+        created_by_id=owner_id,
+        status=Crawl.StatusChoices.SEALED,
+        retry_at=None,
+    )
+    archived_snapshot = Snapshot.objects.create(
+        url="https://example.com",
+        crawl=archived_crawl,
+        status=Snapshot.StatusChoices.SEALED,
+        retry_at=None,
+    )
+    Snapshot.objects.filter(pk=archived_snapshot.pk).update(fs_version=0, retry_at=None)
+    maintenance_crawl = Crawl.objects.create(
+        urls="archivebox://update",
+        created_by_id=owner_id,
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
+    )
+
+    snapshot_ids = CrawlRunner(maintenance_crawl, show_progress=False).load_run_state()
+
+    maintenance_crawl.refresh_from_db()
+    archived_snapshot.refresh_from_db()
+    assert snapshot_ids == []
+    assert maintenance_crawl.snapshot_set.count() == 0
+    assert archived_snapshot.status == Snapshot.StatusChoices.SEALED
+    assert archived_snapshot.retry_at is not None
+
+
 @pytest.mark.django_db(transaction=True)
 def test_crawl_start_event_keeps_retry_at_lease():
     from abx_dl.events import CrawlStartEvent
@@ -616,6 +657,35 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp
     assert snapshot.archiveresult_set.count() == 0
 
 
+@pytest.mark.django_db(transaction=True)
+def test_crawl_runner_preflights_plugins_before_crawl_lifecycle(tmp_path):
+    from abx_dl.events import CrawlEvent, CrawlSetupEvent, InstallEvent
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.services.runner import CrawlRunner
+
+    crawl = Crawl.objects.create(
+        urls="https://example.com",
+        config={
+            "ABXPKG_LIB_DIR": str(tmp_path / "lib"),
+            "PLUGINS": "base",
+            "CHROME_BINARY": "",
+        },
+        created_by_id=get_or_create_system_user_pk(),
+    )
+    runner = CrawlRunner(crawl)
+    event_order = []
+    runner.bus.on(InstallEvent, lambda _event: event_order.append(InstallEvent))
+    runner.bus.on(CrawlEvent, lambda _event: event_order.append(CrawlEvent))
+    runner.bus.on(CrawlSetupEvent, lambda _event: event_order.append(CrawlSetupEvent))
+
+    asyncio.run(runner.run())
+
+    assert event_order.count(InstallEvent) == 1
+    assert event_order.index(InstallEvent) < event_order.index(CrawlEvent)
+    assert event_order.index(InstallEvent) < event_order.index(CrawlSetupEvent)
+
+
 @pytest.mark.django_db(transaction=True)
 def test_crawl_runner_resolves_persona_and_crawl_config_for_each_live_snapshot():
     from abx_dl.events import SnapshotCompletedEvent

archivebox/tests/test_hooks.py

@@ -80,14 +80,14 @@ class TestJSONLParsing:
 
     @staticmethod
     def run_hashes_hook(tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "hash-snapshot"
         output_dir = snap_dir / "hashes"
         output_dir.mkdir(parents=True)
         (snap_dir / "source.txt").write_text("real parser input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -100,7 +100,7 @@ class TestJSONLParsing:
 
     @staticmethod
     def run_parser_hook(tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "parser-snapshot"
         staticfile_dir = snap_dir / "staticfile"
@@ -112,7 +112,7 @@ class TestJSONLParsing:
             encoding="utf-8",
         )
         hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -417,7 +417,7 @@ class TestHookExecution:
 
     @pytest.mark.django_db(transaction=True)
     def test_real_js_hook_runs_through_abxpkg_shebang(self, tmp_path, hermetic_lib_dir):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
         from archivebox.services.runner import run_install
 
         lib_dir = hermetic_lib_dir
@@ -427,7 +427,7 @@ class TestHookExecution:
         snap_dir = crawl_dir / "snapshot"
         hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
 
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             crawl_dir / "chrome",
             config={
@@ -522,7 +522,7 @@ class TestSnapshotHookOutput:
         [(True, "succeeded"), (False, "skipped")],
     )
     def test_hashes_hook_emits_real_archive_result(self, tmp_path, enabled, expected_status):
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / f"snapshot-{expected_status}"
         output_dir = snap_dir / "hashes"
@@ -530,7 +530,7 @@ class TestSnapshotHookOutput:
         (snap_dir / "source.txt").write_text("real hook protocol input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
 
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={
@@ -544,51 +544,21 @@ class TestSnapshotHookOutput:
         process.refresh_from_db()
 
         assert process.exit_code == 0, process.stderr
-        records = extract_records_from_process(process)
+        records = process.get_records()
         assert len(records) == 1
         assert records[0]["type"] == "ArchiveResult"
         assert records[0]["status"] == expected_status
-        assert records[0]["plugin"] == "hashes"
-        assert records[0]["hook_name"] == hook_path.name
-        assert records[0]["plugin_hook"] == str(hook_path)
+        assert process.cmd[0] == str(hook_path)
         if enabled:
             assert (output_dir / "hashes.json").is_file()
         else:
             assert records[0]["output_str"] == "HASHES_ENABLED=False"
 
 
-class TestPluginMetadata:
-    """Test that plugin metadata is added to JSONL records."""
-
-    @pytest.mark.django_db(transaction=True)
-    def test_python_hook_metadata_comes_from_executed_shipped_hook(self, tmp_path):
-        from archivebox.plugins.hooks import extract_records_from_process, run_hook
-
-        snap_dir = tmp_path / "snapshot-metadata"
-        output_dir = snap_dir / "hashes"
-        output_dir.mkdir(parents=True)
-        (snap_dir / "source.txt").write_text("metadata", encoding="utf-8")
-        script = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
-            script,
-            output_dir,
-            config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
-            timeout=30,
-            url="https://example.com/metadata",
-        )
-        process.refresh_from_db()
-
-        assert process.exit_code == 0, process.stderr
-        records = extract_records_from_process(process)
-        assert records[0]["plugin"] == "hashes"
-        assert records[0]["hook_name"] == script.name
-        assert records[0]["plugin_hook"] == str(script)
-
-
 @pytest.mark.django_db(transaction=True)
-def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_path, hermetic_lib_dir):
+def test_abx_dl_hook_execution_exports_singular_node_modules_dir_with_colon_node_path(tmp_path, hermetic_lib_dir):
     """Hook subprocesses must get a real NODE_MODULES_DIR even when NODE_PATH has multiple entries."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import run_install
 
     lib_dir = hermetic_lib_dir
@@ -603,7 +573,7 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
     crawl_dir = tmp_path / "crawl"
     output_dir = crawl_dir / "chrome"
     hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={
@@ -625,16 +595,16 @@ def test_run_hook_exports_singular_node_modules_dir_with_colon_node_path(tmp_pat
 
 
 @pytest.mark.django_db(transaction=True)
-def test_run_hook_executes_python_hooks_through_abxpkg_shebang(tmp_path):
+def test_abx_dl_executes_python_hooks_through_abxpkg_shebang(tmp_path):
     """ArchiveBox treats Python hooks as opaque abxpkg-launched executables."""
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "snapshot"
     output_dir = snap_dir / "hashes"
     output_dir.mkdir(parents=True)
     (snap_dir / "source.txt").write_text("real runtime hook input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={

archivebox/tests/test_machine_models.py

@@ -4,8 +4,8 @@ Unit tests for machine module models: Machine, NetworkInterface, Binary, Process
 Tests cover:
 1. Machine model creation and current() method
 2. NetworkInterface model and network detection
-3. Binary model lifecycle and state machine
-4. Process model lifecycle, hierarchy, and state machine
+3. Binary model lifecycle
+4. Process model lifecycle and hierarchy
 5. JSONL serialization/deserialization
 6. Manager methods
 7. Process tracking methods (replacing pid_utils)
@@ -29,8 +29,6 @@ from archivebox.machine.models import (
     NetworkInterface,
     Binary,
     Process,
-    BinaryMachine,
-    ProcessMachine,
     MACHINE_RECHECK_INTERVAL,
     PID_REUSE_WINDOW,
     PROCESS_TIMEOUT_GRACE,
@@ -509,27 +507,19 @@ class TestBinaryModel:
         assert symlink.resolve() == source.resolve()
 
 
-class TestBinaryStateMachine:
-    """Test the BinaryMachine state machine."""
+class TestBinaryLifecycle:
+    """Test Binary lifecycle prerequisites."""
 
     @pytest.fixture(autouse=True)
     def setup_binary(self, binary):
         self.binary = binary
 
-    def test_binary_state_machine_initial_state(self):
-        """BinaryMachine should start in queued state."""
-        sm = BinaryMachine(self.binary)
-        assert sm.current_state_value == Binary.StatusChoices.QUEUED
-
-    def test_binary_state_machine_can_start(self):
-        """BinaryMachine.can_start() should check name and binproviders."""
-        sm = BinaryMachine(self.binary)
-        assert sm.can_install()
+    def test_binary_can_install_checks_name_and_binproviders(self):
+        assert self.binary.can_install
 
         self.binary.binproviders = ""
         self.binary.save()
-        sm = BinaryMachine(self.binary)
-        assert not sm.can_install()
+        assert not self.binary.can_install
 
 
 class TestProcessModel:
@@ -1002,38 +992,5 @@ class TestProcessClassMethods:
         assert child.exit_code == 143
 
 
-class TestProcessStateMachine:
-    """Test the ProcessMachine state machine."""
-
-    @pytest.fixture(autouse=True)
-    def setup_process(self, process):
-        self.process = process
-
-    def test_process_state_machine_initial_state(self):
-        """ProcessMachine should start in queued state."""
-        sm = ProcessMachine(self.process)
-        assert sm.current_state_value == Process.StatusChoices.QUEUED
-
-    def test_process_state_machine_can_start(self):
-        """ProcessMachine.can_start() should check cmd and machine."""
-        sm = ProcessMachine(self.process)
-        assert sm.can_start()
-
-        self.process.cmd = []
-        self.process.save()
-        sm = ProcessMachine(self.process)
-        assert not sm.can_start()
-
-    def test_process_state_machine_is_exited(self):
-        """ProcessMachine.is_exited() should check exit_code."""
-        sm = ProcessMachine(self.process)
-        assert not sm.is_exited()
-
-        self.process.exit_code = 0
-        self.process.save()
-        sm = ProcessMachine(self.process)
-        assert sm.is_exited()
-
-
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

archivebox/tests/test_machine_service.py

@@ -13,6 +13,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
 def _runtime_env(data_dir: Path, **extra: str) -> dict[str, str]:
     return {
         "ABXPKG_LIB_DIR": str(data_dir / "lib"),
+        "PLUGINS": "liteparse",
         "LITEPARSE_ENABLED": "True",
         "TIMEOUT": "180",
         "ABXPKG_INSTALL_TIMEOUT": "180",

archivebox/tests/test_migration_archiveresult_unique_plugin.py

@@ -1,53 +0,0 @@
-import pytest
-from django.db import connection
-from django.db.migrations.executor import MigrationExecutor
-
-
-pytestmark = pytest.mark.django_db(transaction=True)
-
-
-def test_migration_consolidates_plugin_rows_and_enforces_uniqueness():
-    try:
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0051_postgres_url_pattern_ops_index")])
-        old_apps = executor.loader.project_state([("core", "0051_postgres_url_pattern_ops_index")]).apps
-        Crawl = old_apps.get_model("crawls", "Crawl")
-        Snapshot = old_apps.get_model("core", "Snapshot")
-        ArchiveResult = old_apps.get_model("core", "ArchiveResult")
-        crawl = Crawl.objects.create(urls="https://example.com")
-        snapshot = Snapshot.objects.create(url="https://example.com/migration", crawl=crawl, output_size=10)
-        ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="responses",
-            hook_name="browser-upload",
-            status="succeeded",
-            output_files={"browser.png": {"size": 7}},
-            output_size=7,
-            output_mimetypes="image/png",
-        )
-        ArchiveResult.objects.create(
-            snapshot=snapshot,
-            plugin="responses",
-            hook_name="server-capture",
-            status="noresults",
-            output_files={"server.json": {"size": 3}},
-            output_size=3,
-            output_mimetypes="application/json",
-        )
-
-        executor = MigrationExecutor(connection)
-        executor.migrate([("core", "0052_unique_archiveresult_per_snapshot_plugin")])
-        new_apps = executor.loader.project_state([("core", "0052_unique_archiveresult_per_snapshot_plugin")]).apps
-        Snapshot = new_apps.get_model("core", "Snapshot")
-        ArchiveResult = new_apps.get_model("core", "ArchiveResult")
-        snapshot = Snapshot.objects.get(id=snapshot.id)
-        result = ArchiveResult.objects.get(snapshot=snapshot, plugin="responses")
-
-        assert ArchiveResult.objects.filter(snapshot=snapshot, plugin="responses").count() == 1
-        assert result.status == "succeeded"
-        assert set(result.output_files) == {"browser.png", "server.json"}
-        assert result.output_size == 10
-        assert snapshot.output_size == 10
-        assert set(result.output_mimetypes.split(",")) == {"application/json", "image/png"}
-    finally:
-        MigrationExecutor(connection).migrate([("core", "0052_unique_archiveresult_per_snapshot_plugin")])

archivebox/tests/test_migrations_07_to_09.py

@@ -403,7 +403,7 @@ def test_update_saves_migrated_snapshots_without_foreign_key_errors(archive_07):
     result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
     assert result.returncode == 0, f"Init failed: {result.stderr}"
 
-    result = run_archivebox_migration_cmd(work_dir, ["update"], timeout=60)
+    result = run_archivebox_migration_cmd(work_dir, ["update", "--migrate-only"], timeout=60)
     output = result.stdout + result.stderr
     assert result.returncode == 0, f"Update failed after migration: {result.stderr}"
     assert "FOREIGN KEY constraint failed" not in output

archivebox/tests/test_migrations_08_to_09.py

@@ -6,7 +6,7 @@ Migration tests from 0.8.x to 0.9.x.
 - Crawl model for grouping URLs
 - Seed model (removed in 0.9.x)
 - UUID primary keys for Snapshot
-- Status fields for state machine
+- Status fields for queued lifecycle processing
 - New fields like depth, retry_at, etc.
 """
 
@@ -898,7 +898,10 @@ def test_update_preserves_legacy_plugin_directory_without_output_files(migration
         (snapshot["id"],),
     ).fetchall()
     conn.close()
-    assert rows == [("", "{}", "")]
+    assert len(rows) == 2
+    assert all(output_str == "" for output_str, _output_files, _hook_name in rows)
+    assert len({hook_name for _output_str, _output_files, hook_name in rows}) == 2
+    assert sum(not hook_name for _output_str, _output_files, hook_name in rows) == 1
 
 
 def test_07_filesystem_hop_preserves_complete_output_tree(tmp_path):

archivebox/tests/test_process_runtime_paths.py

@@ -11,14 +11,14 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 class TestProcessRuntimePaths:
     def test_hook_processes_use_isolated_runtime_dir(self, tmp_path):
-        from archivebox.plugins.hooks import run_hook
+        from archivebox.tests.conftest import run_test_hook
 
         snap_dir = tmp_path / "snapshot"
         output_dir = snap_dir / "hashes"
         output_dir.mkdir(parents=True)
         (snap_dir / "source.txt").write_text("real runtime path input", encoding="utf-8")
         hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-        process = run_hook(
+        process = run_test_hook(
             hook_path,
             output_dir,
             config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_recursive_crawl.py

@@ -493,10 +493,10 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
 ):
     """Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
 
-    from abx_dl.models import discover_plugins
+    from archivebox.plugins.discovery import get_plugin_catalog
 
     root_url = recursive_test_site["root_url"]
-    plugin_selection = ",".join(sorted(plugin for plugin in discover_plugins().keys() if not plugin.startswith("claude")))
+    plugin_selection = ",".join(sorted(plugin for plugin in get_plugin_catalog() if not plugin.startswith("claude")))
     env = os.environ.copy()
     for preinstalled_path_key in (
         "CHROME_BINARY",

archivebox/tests/test_search.py

@@ -118,8 +118,8 @@ def configure_ripgrep_search_backend(lib_dir: Path) -> None:
     )
 
 
-def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
-    from archivebox.search.backends import search_backend_env
+def test_search_backend_command_env_serializes_config_without_mutating_process_env(tmp_path):
+    from archivebox.search.backends import search_backend_command_env
 
     old_env = os.environ.get("SEARCH_BACKEND_SONIC_HOST_NAME")
     os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] = "old-host"
@@ -136,15 +136,14 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
     )
 
     try:
-        with search_backend_env(config=config):
-            assert os.environ["SEARCH_BACKEND_ENGINE"] == "sonic"
-            assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
-            assert os.environ["SEARCH_BACKEND_SONIC_PORT"] == "1491"
-            assert os.environ["SEARCH_BACKEND_SONIC_PASSWORD"] == "SecretPassword"
-            assert os.environ["RIPGREP_BINARY"] == str(tmp_path / "env" / "bin" / "rg")
-            assert "IGNORED_NONE_VALUE" not in os.environ
-            assert "UNRELATED_BINARY_NAME" not in os.environ
-
+        env = search_backend_command_env(config=config)
+        assert env["SEARCH_BACKEND_ENGINE"] == "sonic"
+        assert env["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
+        assert env["SEARCH_BACKEND_SONIC_PORT"] == "1491"
+        assert env["SEARCH_BACKEND_SONIC_PASSWORD"] == "SecretPassword"
+        assert env["RIPGREP_BINARY"] == str(tmp_path / "env" / "bin" / "rg")
+        assert "IGNORED_NONE_VALUE" not in env
+        assert env["UNRELATED_BINARY_NAME"] == "rg"
         assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "old-host"
     finally:
         if old_env is None:

archivebox/tests/test_snapshot_service.py

@@ -40,7 +40,7 @@ def _snapshot_state(cwd: Path, url: str) -> dict[str, object]:
         }
 
 
-def test_snapshot_merge_consolidates_plugin_output_identity(admin_user):
+def test_snapshot_merge_consolidates_only_exact_hook_identity(admin_user):
     from archivebox.crawls.models import Crawl
 
     keeper_crawl = Crawl.objects.create(urls="https://example.com", created_by=admin_user)
@@ -64,15 +64,24 @@ def test_snapshot_merge_consolidates_plugin_output_identity(admin_user):
         output_files={"archivewebpage.wacz": {"size": 7}},
         output_size=7,
     )
+    ArchiveResult.objects.create(
+        snapshot=duplicate,
+        plugin="archivewebpage",
+        hook_name="on_Snapshot__16_archivewebpage_start",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_str="recording started",
+    )
+
     Snapshot._merge_snapshots([keeper, duplicate])
 
     assert not Snapshot.objects.filter(pk=duplicate.pk).exists()
     results = ArchiveResult.objects.filter(snapshot=keeper, plugin="archivewebpage")
-    assert results.count() == 1
-    stop_result = results.get()
+    assert results.count() == 2
+    stop_result = results.get(hook_name="on_Snapshot__65_archivewebpage_stop")
     assert stop_result.status == ArchiveResult.StatusChoices.SUCCEEDED
     assert stop_result.output_str == "archivewebpage.wacz"
     assert set(stop_result.output_files) == {"older.wacz", "archivewebpage.wacz"}
+    assert results.filter(hook_name="on_Snapshot__16_archivewebpage_start").exists()
 
 
 @pytest.mark.timeout(180)

archivebox/tests/test_ui_admin_archiveresult.py

@@ -142,8 +142,7 @@ class TestArchiveResultAdminListView:
         snapshot.refresh_from_db()
         assert snapshot.output_size == 0
 
-    def test_duplicate_plugin_result_is_rejected_without_touching_output(self, snapshot):
-        from django.db import IntegrityError, transaction
+    def test_deleting_sibling_hook_preserves_shared_plugin_output(self, snapshot):
         from archivebox.core.models import ArchiveResult
 
         output_dir = Path(snapshot.output_dir) / "responses"
@@ -157,13 +156,14 @@ class TestArchiveResultAdminListView:
             status=ArchiveResult.StatusChoices.SUCCEEDED,
             output_size=18,
         )
-        with pytest.raises(IntegrityError), transaction.atomic():
-            ArchiveResult.objects.create(
-                snapshot=snapshot,
-                plugin="responses",
-                hook_name="on_Snapshot__24_responses.daemon.bg.replayed",
-                status=ArchiveResult.StatusChoices.NORESULTS,
-            )
+        duplicate = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="responses",
+            hook_name="on_Snapshot__24_responses.daemon.bg.replayed",
+            status=ArchiveResult.StatusChoices.NORESULTS,
+        )
+
+        duplicate.delete()
 
         assert ArchiveResult.objects.filter(pk=primary.pk).exists()
         assert output_file.read_text() == "captured response\n"

archivebox/tests/test_ui_admin_links.py

@@ -15,13 +15,13 @@ from archivebox.tests.conftest import cli_env, run_archivebox_cmd
 from archivebox.tests.conftest import install_real_binary
 
 
-pytestmark = pytest.mark.django_db
+pytestmark = pytest.mark.django_db(transaction=True)
 
 
 @pytest.fixture
 def real_hook_result(tmp_path):
     from archivebox.core.models import ArchiveResult
-    from archivebox.plugins.hooks import extract_records_from_process, run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snapshot = _create_snapshot()
     snap_dir = Path(snapshot.output_dir)
@@ -29,7 +29,7 @@ def real_hook_result(tmp_path):
     output_dir.mkdir(parents=True, exist_ok=True)
     (snap_dir / "source.txt").write_text("real admin link hook input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={
@@ -45,12 +45,12 @@ def real_hook_result(tmp_path):
     )
     process.refresh_from_db()
     assert process.exit_code == 0, process.stderr
-    record = extract_records_from_process(process)[0]
+    record = process.get_records()[0]
     hashes_file = output_dir / "hashes.json"
     result = ArchiveResult.objects.create(
         snapshot=snapshot,
-        plugin=record["plugin"],
-        hook_name=record["hook_name"],
+        plugin="hashes",
+        hook_name=hook_path.name,
         process=process,
         status=record["status"],
         output_str=record["output_str"],

archivebox/tests/test_ui_admin_machine.py

@@ -19,14 +19,14 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 @pytest.fixture
 def real_exited_hook_process(tmp_path):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "snapshot"
     output_dir = snap_dir / "hashes"
     output_dir.mkdir(parents=True)
     (snap_dir / "source.txt").write_text("real admin hook input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},

archivebox/tests/test_ui_admin_snapshot.py

@@ -324,6 +324,35 @@ def test_snapshot_admin_preview_uses_extension_screenshot_when_standard_screensh
     assert "chrome_extension_screenshot/screenshot-2.png" not in preview["fallback_list"]
 
 
+def test_snapshot_admin_attributes_new_tags_to_authenticated_user(client, snapshot, admin_user):
+    from archivebox.core.models import Tag
+
+    client.force_login(admin_user)
+    response = client.post(
+        reverse("admin:core_snapshot_change", args=[snapshot.pk]),
+        {
+            "url": snapshot.url,
+            "title": snapshot.title or "",
+            "tags_editor": "admin-created-tag",
+            "permissions_config": "private",
+            "status": snapshot.status,
+            "retry_at": "",
+            "bookmarked_at_0": snapshot.bookmarked_at.date().isoformat(),
+            "bookmarked_at_1": snapshot.bookmarked_at.time().isoformat(),
+            "crawl": str(snapshot.crawl_id),
+            "config": '{"SAVE_ARCHIVE_DOT_ORG": "false"}',
+            "notes": "",
+            "_save": "Save",
+        },
+        HTTP_HOST=ADMIN_TEST_HOST,
+    )
+
+    assert response.status_code == 302, response.context and response.context["adminform"].form.errors
+    tag = Tag.objects.get(name="admin-created-tag")
+    assert tag.created_by == admin_user
+    assert snapshot.tags.filter(pk=tag.pk).exists()
+
+
 class TestSnapshotProgressStats:
     """Tests for Snapshot.get_progress_stats() method."""
 
@@ -485,10 +514,46 @@ class TestSnapshotProgressStats:
 
         assert _count_media_files(result) == 2
         assert _list_media_files(result) == [
-            {"name": "audio.mp3", "path": "ytdlp/audio.mp3", "size": 222},
-            {"name": "video.mp4", "path": "ytdlp/video.mp4", "size": 111},
+            {
+                "name": "video.mp4",
+                "path": "ytdlp/video.mp4",
+                "size": 111,
+                "media_type": "video",
+                "is_video": True,
+                "is_audio": False,
+                "is_browser_playable": True,
+            },
+            {
+                "name": "audio.mp3",
+                "path": "ytdlp/audio.mp3",
+                "size": 222,
+                "media_type": "audio",
+                "is_video": False,
+                "is_audio": True,
+                "is_browser_playable": True,
+            },
         ]
 
+    def test_ytdlp_discover_outputs_prefers_browser_playable_video(self, snapshot):
+        from archivebox.core.models import ArchiveResult
+
+        ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="ytdlp",
+            status="succeeded",
+            output_files={
+                "thumbnail.jpg": {"size": 20_000, "mimetype": "image/jpeg", "extension": "jpg"},
+                "large.mkv": {"size": 10_000, "mimetype": "video/x-matroska", "extension": "mkv"},
+                "small.mp4": {"size": 111, "mimetype": "video/mp4", "extension": "mp4"},
+            },
+            output_size=30_111,
+        )
+
+        outputs = snapshot.discover_outputs(include_filesystem_fallback=False)
+        ytdlp_output = next(output for output in outputs if output["name"] == "ytdlp")
+
+        assert ytdlp_output["path"] == "ytdlp/small.mp4"
+
     def test_discover_outputs_falls_back_to_hashes_index_without_filesystem_walk(
         self,
         snapshot,
@@ -928,10 +993,9 @@ class TestSnapshotOutputDeletion:
         from archivebox.core.models import ArchiveResult
 
         first = self._create_output(snapshot, size=11)
-        second = self._create_output(snapshot, plugin="singlefile", hook_name="on_Snapshot__50_singlefile.py", size=13)
+        second = self._create_output(snapshot, hook_name="on_Snapshot__51_screenshot_retry.py", size=13)
         kept = self._create_output(snapshot, plugin="pdf", hook_name="on_Snapshot__60_pdf.py", size=7)
         deleted_dir = Path(first.output_dir)
-        second_deleted_dir = Path(second.output_dir)
         kept_dir = Path(kept.output_dir)
         hashes_dir = Path(snapshot.output_dir) / "hashes"
         hashes_dir.mkdir(parents=True, exist_ok=True)
@@ -964,7 +1028,6 @@ class TestSnapshotOutputDeletion:
         assert not ArchiveResult.objects.filter(pk__in=[first.pk, second.pk]).exists()
         assert ArchiveResult.objects.filter(pk=kept.pk).exists()
         assert not deleted_dir.exists()
-        assert not second_deleted_dir.exists()
         assert kept_dir.exists()
         snapshot.refresh_from_db()
         assert snapshot.output_size == 7

archivebox/tests/test_ui_live_progress.py

@@ -21,14 +21,14 @@ pytestmark = pytest.mark.django_db(transaction=True)
 
 @pytest.fixture
 def real_unscoped_hook_process(tmp_path):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = tmp_path / "snapshot"
     output_dir = snap_dir / "hashes"
     output_dir.mkdir(parents=True)
     (snap_dir / "source.txt").write_text("real live progress input", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.hashes").joinpath("on_Snapshot__93_hashes.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -55,7 +55,7 @@ def real_snapshot_hook_projection(snapshot, cached_abxpkg_lib_dir):
 
 @pytest.fixture
 def real_second_snapshot_hook_process(snapshot, tmp_path):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
 
     snap_dir = Path(snapshot.output_dir)
     staticfile_dir = snap_dir / "staticfile"
@@ -64,7 +64,7 @@ def real_second_snapshot_hook_process(snapshot, tmp_path):
     output_dir.mkdir(parents=True, exist_ok=True)
     (staticfile_dir / "input.txt").write_text("plain text without links", encoding="utf-8")
     hook_path = Path(str(files("abx_plugins.plugins.parse_txt_urls").joinpath("on_Snapshot__71_parse_txt_urls.py")))
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={"ABXPKG_LIB_DIR": str(tmp_path / "lib"), "SNAP_DIR": str(snap_dir)},
@@ -78,7 +78,7 @@ def real_second_snapshot_hook_process(snapshot, tmp_path):
 
 @pytest.fixture
 def real_crawl_setup_process(snapshot, hermetic_lib_dir):
-    from archivebox.plugins.hooks import run_hook
+    from archivebox.tests.conftest import run_test_hook
     from archivebox.services.runner import run_install
 
     hook_path = Path(str(files("abx_plugins.plugins.chrome").joinpath("on_CrawlSetup__89_chrome_kill_zombies.js")))
@@ -86,7 +86,7 @@ def real_crawl_setup_process(snapshot, hermetic_lib_dir):
     run_install(plugin_names=["chrome"])
     binary_env = resolve_abxpkg_binary_env(hermetic_lib_dir, deps_from=config_path)
     output_dir = Path(snapshot.crawl.output_dir) / "chrome"
-    process = run_hook(
+    process = run_test_hook(
         hook_path,
         output_dir,
         config={

archivebox/tests/test_urls.py

@@ -15,22 +15,26 @@ REPO_ROOT = Path(__file__).resolve().parents[2]
 
 
 @pytest.mark.django_db
-def test_archiveresult_relpath_uses_plugin_result_that_owns_output(admin_user):
+def test_archiveresult_relpath_uses_sibling_hook_that_owns_output(admin_user):
     from archivebox.core.models import ArchiveResult, Snapshot
     from archivebox.core.views import _resolve_archiveresult_relpath
     from archivebox.crawls.models import Crawl
 
     crawl = Crawl.objects.create(urls="https://example.com", created_by=admin_user)
     snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl)
+    ArchiveResult.objects.create(
+        snapshot=snapshot,
+        plugin="screenshot",
+        hook_name="on_Snapshot__archivebox_browser_extension_upload",
+        status=ArchiveResult.StatusChoices.SUCCEEDED,
+        output_files={"browser.png": {"size": 7}},
+    )
     server_result = ArchiveResult.objects.create(
         snapshot=snapshot,
         plugin="screenshot",
         hook_name="on_Snapshot__50_screenshot",
         status=ArchiveResult.StatusChoices.SUCCEEDED,
-        output_files={
-            "browser.png": {"size": 7},
-            "screenshot.png": {"size": 6, "root_relative": True},
-        },
+        output_files={"screenshot.png": {"size": 6, "root_relative": True}},
     )
 
     resolved_path, result = _resolve_archiveresult_relpath(snapshot, "screenshot/screenshot.png")
@@ -134,6 +138,8 @@ def test_static_html_and_markdown_preview_images_rewrite_to_captured_responses(t
     assert "ETag" not in response.headers
     assert "max-age=60" in response.headers["Cache-Control"]
     assert b"archivebox-static-html-preview-style" in response.content
+    assert b"width: min(100%, 72rem)" in response.content
+    assert b"min-height: 100vh" in response.content
     assert b"img:not([width]):not([height])" in response.content
     assert b"a > img:not([width]):not([height])" in response.content
     assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content
@@ -163,6 +169,8 @@ def test_static_html_and_markdown_preview_images_rewrite_to_captured_responses(t
     assert "ETag" not in response.headers
     assert "max-age=60" in response.headers["Cache-Control"]
     assert b"archivebox-static-html-preview-style" in response.content
+    assert b"width: min(100%, 72rem)" in response.content
+    assert b"min-height: 100vh" in response.content
     assert b'src="../responses/all/20260722T061544__GET__https_3A_2F_2Fsweeting.me_2Fimages_2Ftwitter.png"' in response.content
 
 

archivebox/workers/models.py

@@ -3,22 +3,15 @@ __package__ = "archivebox.workers"
 import inspect
 import logging
 
-from typing import Any, ClassVar, Protocol, cast
 from collections.abc import Iterable
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
-from statemachine.callbacks import SPECS_ALL
-from statemachine.dispatcher import Listener, Listeners
-from statemachine.graph import iterate_states_and_transitions
+from typing import Any, ClassVar, cast
 
 from django.db import models
-from django.core import checks
 from django.utils import timezone
-from django.utils.functional import classproperty
 from django_stubs_ext.db.models import TypedModelMeta
 
-from statemachine import registry, StateMachine, State
-
 
 class DefaultStatusChoices(models.TextChoices):
     QUEUED = "queued", "Queued"
@@ -43,230 +36,59 @@ MODULE_PATH = Path(__file__).resolve()
 REPO_ROOT = MODULE_PATH.parents[2]
 PACKAGE_ROOT = MODULE_PATH.parents[1]
 
-ObjectState = State | str
-ObjectStateList = Iterable[ObjectState]
 
+class ModelWithQueue(models.Model):
+    """Durable queue fields and atomic lease operations shared by work rows.
 
-class ModelStateMachine(Protocol):
-    def tick(self) -> Any: ...
+    Concrete models own lifecycle transitions. This mixin only owns the common
+    database queue protocol: status, retry_at, pause/resume, and claims.
+    """
 
-    def pause_requested(self) -> Any: ...
-
-    def resume_requested(self) -> Any: ...
-
-
-class BaseModelWithStateMachine(models.Model):
-    StatusChoices: ClassVar[type[DefaultStatusChoices]]
-
-    # status: models.CharField
-    # retry_at: models.DateTimeField
-
-    state_machine_name: str | None = None
-    state_field_name: str
-    state_machine_attr: str = "sm"
-    bind_events_as_methods: bool = False
+    StatusChoices: ClassVar[type[models.TextChoices]] = DefaultStatusChoices
+    INITIAL_STATE: ClassVar[str] = DefaultStatusChoices.QUEUED
+    ACTIVE_STATE: ClassVar[str] = DefaultStatusChoices.STARTED
+    FINAL_STATES: ClassVar[tuple[str, ...]] = (DefaultStatusChoices.SEALED,)
     warn_on_save_outside_runner: ClassVar[bool] = True
 
-    active_state: ObjectState
-    retry_at_field_name: str
+    status: models.CharField = models.CharField(**default_status_field.deconstruct()[3])
+    retry_at: models.DateTimeField = models.DateTimeField(**default_retry_at_field.deconstruct()[3])
 
     class Meta(TypedModelMeta):
         app_label = "workers"
         abstract = True
 
-    @property
-    def sm(self) -> StateMachine:
-        """Build the python-statemachine wrapper only at transition callsites.
-
-        This model is loaded by high-volume paths that do not drive lifecycle
-        transitions: admin lists, progress polling, index-only maintenance, and
-        bulk recovery scans all instantiate thousands of rows just to read or
-        update ordinary columns.  python-statemachine setup is correct but not
-        free: it creates per-instance state wrappers, callback registries,
-        queues, locks, and callback adapters.  Paying that cost from Django's
-        model __init__ made plain ORM materialization scale with state-machine
-        setup instead of row decoding.
-
-        ArchiveBox drives lifecycle transitions explicitly through `.sm`
-        (`snapshot.sm.tick()`, `crawl.sm.seal()`, etc.), so the machine can be
-        cached on first use without changing the state model.  Code that only
-        needs database fields never constructs one.
-        """
-        try:
-            machine = vars(self)["_archivebox_state_machine"]
-        except KeyError:
-            machine = self.StateMachineClass(self, state_field=self.state_field_name)
-            vars(self)["_archivebox_state_machine"] = machine
-        return cast(StateMachine, machine)
+    FINAL_OR_ACTIVE_STATES: ClassVar[tuple[str, ...]] = (*FINAL_STATES, ACTIVE_STATE)
 
     @classmethod
     def status_counts(cls, queryset: models.QuerySet | None = None, statuses: Iterable[str] | None = None) -> dict[str, int]:
-        """Count requested statuses with separate indexed COUNT probes.
-
-        For live/progress views this is often faster on large SQLite data dirs
-        than a grouped aggregate, because each status can use the status index
-        directly and the caller usually needs only a few states.
-        """
         qs = queryset if queryset is not None else cls.objects.all()
         return {status: qs.filter(status=status).count() for status in (statuses or cls.StatusChoices.values)}
 
-    @classmethod
-    def check(cls, sender=None, **kwargs):
-        import sys
-
-        # Skip state machine checks during makemigrations to avoid premature registry access
-        if "makemigrations" in sys.argv:
-            return super().check(**kwargs)
-
-        errors = super().check(**kwargs)
-
-        found_id_field = False
-        found_status_field = False
-        found_retry_at_field = False
-
-        for field in cls._meta.get_fields():
-            if getattr(field, "_is_state_field", False):
-                if cls.state_field_name == field.name:
-                    found_status_field = True
-                    if getattr(field, "choices", None) != cls.StatusChoices.choices:
-                        errors.append(
-                            checks.Error(
-                                f"{cls.__name__}.{field.name} must have choices set to {cls.__name__}.StatusChoices.choices",
-                                hint=f"{cls.__name__}.{field.name}.choices = {getattr(field, 'choices', None)!r}",
-                                obj=cls,
-                                id="workers.E011",
-                            ),
-                        )
-            if getattr(field, "_is_retry_at_field", False):
-                if cls.retry_at_field_name == field.name:
-                    found_retry_at_field = True
-            if field.name == "id" and getattr(field, "primary_key", False):
-                found_id_field = True
-
-        if not found_status_field:
-            errors.append(
-                checks.Error(
-                    f"{cls.__name__}.state_field_name must be defined and point to a StatusField()",
-                    hint=f"{cls.__name__}.state_field_name = {cls.state_field_name!r} but {cls.__name__}.{cls.state_field_name!r} was not found or does not refer to StatusField",
-                    obj=cls,
-                    id="workers.E012",
-                ),
-            )
-        if not found_retry_at_field:
-            errors.append(
-                checks.Error(
-                    f"{cls.__name__}.retry_at_field_name must be defined and point to a RetryAtField()",
-                    hint=f"{cls.__name__}.retry_at_field_name = {cls.retry_at_field_name!r} but {cls.__name__}.{cls.retry_at_field_name!r} was not found or does not refer to RetryAtField",
-                    obj=cls,
-                    id="workers.E013",
-                ),
-            )
-
-        if not found_id_field:
-            errors.append(
-                checks.Error(
-                    f"{cls.__name__} must have an id field that is a primary key",
-                    hint=f"{cls.__name__}.id field missing or not configured as primary key",
-                    obj=cls,
-                    id="workers.E014",
-                ),
-            )
-
-        if not isinstance(cls.state_machine_name, str):
-            errors.append(
-                checks.Error(
-                    f"{cls.__name__}.state_machine_name must be a dotted-import path to a StateMachine class",
-                    hint=f"{cls.__name__}.state_machine_name = {cls.state_machine_name!r}",
-                    obj=cls,
-                    id="workers.E015",
-                ),
-            )
-
-        try:
-            cls.StateMachineClass
-        except Exception as err:
-            errors.append(
-                checks.Error(
-                    f"{cls.__name__}.state_machine_name must point to a valid StateMachine class, but got {type(err).__name__} {err} when trying to access {cls.__name__}.StateMachineClass",
-                    hint=f"{cls.__name__}.state_machine_name = {cls.state_machine_name!r}",
-                    obj=cls,
-                    id="workers.E016",
-                ),
-            )
-
-        if cls.INITIAL_STATE not in cls.StatusChoices.values:
-            errors.append(
-                checks.Error(
-                    f"{cls.__name__}.StateMachineClass.initial_state must be present within {cls.__name__}.StatusChoices",
-                    hint=f"{cls.__name__}.StateMachineClass.initial_state = {cls.StateMachineClass.initial_state!r}",
-                    obj=cls,
-                    id="workers.E017",
-                ),
-            )
-
-        if cls.ACTIVE_STATE not in cls.StatusChoices.values:
-            errors.append(
-                checks.Error(
-                    f"{cls.__name__}.active_state must be set to a valid State present within {cls.__name__}.StatusChoices",
-                    hint=f"{cls.__name__}.active_state = {cls.active_state!r}",
-                    obj=cls,
-                    id="workers.E018",
-                ),
-            )
-
-        for state in cls.FINAL_STATES:
-            if state not in cls.StatusChoices.values:
-                errors.append(
-                    checks.Error(
-                        f"{cls.__name__}.StateMachineClass.final_states must all be present within {cls.__name__}.StatusChoices",
-                        hint=f"{cls.__name__}.StateMachineClass.final_states = {cls.StateMachineClass.final_states!r}",
-                        obj=cls,
-                        id="workers.E019",
-                    ),
-                )
-                break
-        return errors
-
-    @staticmethod
-    def _state_to_str(state: ObjectState) -> str:
-        """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
-        return str(state.value) if isinstance(state, State) else str(state)
-
     @property
-    def RETRY_AT(self) -> datetime:
-        return getattr(self, self.retry_at_field_name)
+    def RETRY_AT(self) -> datetime | None:
+        return self.retry_at
 
     @RETRY_AT.setter
-    def RETRY_AT(self, value: datetime):
-        setattr(self, self.retry_at_field_name, value)
+    def RETRY_AT(self, value: datetime | None) -> None:
+        self.retry_at = value
 
     @property
     def STATE(self) -> str:
-        return getattr(self, self.state_field_name)
+        return self.status
 
     @STATE.setter
-    def STATE(self, value: str):
-        setattr(self, self.state_field_name, value)
+    def STATE(self, value: str) -> None:
+        self.status = value
 
-    def bump_retry_at(self, seconds: int = 10):
-        self.RETRY_AT = timezone.now() + timedelta(seconds=seconds)
+    def bump_retry_at(self, seconds: int = 10) -> None:
+        self.retry_at = timezone.now() + timedelta(seconds=seconds)
 
     @property
     def is_paused(self) -> bool:
         paused_state = getattr(self.StatusChoices, "PAUSED", None)
-        return paused_state is not None and self.STATE == paused_state
+        return paused_state is not None and self.status == paused_state
 
     def safe_update(self, update_fields: dict[str, Any], *, refresh: bool = True, extra_filter: dict[str, Any] | None = None) -> bool:
-        """
-        Atomic single-row UPDATE for scheduler writes that bypass save().
-
-        The write is unconditional unless the caller passes extra_filter — the
-        previous implicit modified_at CAS predicate spuriously collided with
-        concurrent writers to unrelated fields (every save bumps modified_at),
-        which silently dropped state-machine transitions. Callers that need a
-        transition guard (only advance from state A to state B; only requeue a
-        row still holding lease X) pass extra_filter explicitly.
-        """
         values = dict(update_fields)
         values.setdefault("modified_at", timezone.now())
         queryset = type(self).objects.filter(pk=self.pk)
@@ -274,17 +96,14 @@ class BaseModelWithStateMachine(models.Model):
             queryset = queryset.filter(**extra_filter)
         updated = queryset.update(**values)
         if updated != 1 and extra_filter:
-            current = type(self).objects.filter(pk=self.pk).values(self.state_field_name).first()
-            current_status = current.get(self.state_field_name) if current else "<deleted>"
+            current = type(self).objects.filter(pk=self.pk).values("status").first()
             logger.info(
-                "SafeUpdateGuardMiss: %s row %s extra_filter=%s did not match (current %s=%s, loaded %s=%s); update_fields=%s skipped",
+                "SafeUpdateGuardMiss: %s row %s extra_filter=%s current_status=%s loaded_status=%s update_fields=%s skipped",
                 type(self).__name__,
                 self.pk,
                 extra_filter,
-                self.state_field_name,
-                current_status,
-                self.state_field_name,
-                self.STATE,
+                current.get("status") if current else "<deleted>",
+                self.status,
                 sorted(values),
             )
         if refresh:
@@ -294,7 +113,7 @@ class BaseModelWithStateMachine(models.Model):
                 pass
         return updated == 1
 
-    def save(self, *args, **kwargs):
+    def save(self, *args: Any, **kwargs: Any) -> None:
         from archivebox.machine.models import Process
 
         process = Process.current()
@@ -325,12 +144,11 @@ class BaseModelWithStateMachine(models.Model):
                 finally:
                     del frame
                 logger.warning(
-                    "%s.save() outside runner process: id=%s status=%s retry_at=%s process=%s root=%s caller=%s; "
-                    "queue/status writes outside the runner should usually use safe_update()",
+                    "%s.save() outside runner process: id=%s status=%s retry_at=%s process=%s root=%s caller=%s",
                     type(self).__name__,
                     self.pk,
-                    self.STATE,
-                    self.RETRY_AT,
+                    self.status,
+                    self.retry_at,
                     process.process_type,
                     root_type,
                     caller,
@@ -338,327 +156,74 @@ class BaseModelWithStateMachine(models.Model):
         super().save(*args, **kwargs)
 
     def pause(self, *, save: bool = True) -> bool:
-        try:
-            paused_state = self.StatusChoices.PAUSED
-        except AttributeError:
-            return False
-        if self.STATE in self.FINAL_STATES or self.is_paused:
+        paused_state = getattr(self.StatusChoices, "PAUSED", None)
+        if paused_state is None or self.status in self.FINAL_STATES or self.is_paused:
             return False
+        previous_status = self.status
+        self.status = paused_state
+        self.retry_at = RETRY_AT_MAX
         if save:
-            cast(ModelStateMachine, self.sm).pause_requested()
-            self.refresh_from_db()
-            return self.is_paused
-        self.STATE = paused_state
-        self.RETRY_AT = RETRY_AT_MAX
+            return self.safe_update(
+                {"status": paused_state, "retry_at": RETRY_AT_MAX},
+                extra_filter={"status": previous_status},
+            )
         return True
 
     def resume(self, *, when: datetime | None = None, save: bool = True) -> bool:
-        try:
-            paused_state = self.StatusChoices.PAUSED
-        except AttributeError:
-            return False
-        if not self.is_paused:
+        paused_state = getattr(self.StatusChoices, "PAUSED", None)
+        if paused_state is None or not self.is_paused:
             return False
+        resume_at = when or timezone.now()
+        self.status = self.StatusChoices.QUEUED
+        self.retry_at = resume_at
         if save:
-            if when is None:
-                cast(ModelStateMachine, self.sm).resume_requested()
-                self.refresh_from_db()
-                return self.STATE == self.StatusChoices.QUEUED
-            self.STATE = self.StatusChoices.QUEUED
-            self.RETRY_AT = when or timezone.now()
-            updated = self.safe_update(
-                {
-                    self.state_field_name: self.StatusChoices.QUEUED,
-                    self.retry_at_field_name: self.RETRY_AT,
-                },
-                extra_filter={self.state_field_name: paused_state},
+            return self.safe_update(
+                {"status": self.StatusChoices.QUEUED, "retry_at": resume_at},
+                extra_filter={"status": paused_state},
             )
-            return updated
-        self.STATE = self.StatusChoices.QUEUED
-        self.RETRY_AT = when or timezone.now()
         return True
 
-    def update_and_requeue(self, *, refresh: bool = True, **kwargs) -> bool:
-        """
-        Scheduler-facing wrapper around safe_update().
-
-        Call this when a state-machine row should become visible to the
-        runner. It preserves the current retry_at lease as an additional guard
-        while safe_update() owns the modified_at CAS write and refresh.
-        """
-        # retry_at is the scheduler lease, but it is not enough by itself:
-        # sealed maintenance rows can legitimately keep the same retry_at while
-        # other fields change. Include modified_at as a cheap compare-and-swap
-        # guard so iterator/recovery scans never overwrite a row that the
-        # runner touched after the object was read.
-        current_retry_at = self.RETRY_AT
-        return self.safe_update(
-            dict(kwargs),
-            refresh=refresh,
-            extra_filter={self.retry_at_field_name: current_retry_at},
-        )
+    def update_and_requeue(self, *, refresh: bool = True, **kwargs: Any) -> bool:
+        return self.safe_update(dict(kwargs), refresh=refresh, extra_filter={"retry_at": self.retry_at})
 
     @classmethod
     def get_queue(cls):
-        """
-        Get the sorted and filtered QuerySet of objects that are ready for processing.
-        retry_at is the only scheduler signal; callers branch on status after selection.
-        """
-        return cls.objects.filter(
-            retry_at__lte=timezone.now(),
-        ).order_by("retry_at")
+        return cls.objects.filter(retry_at__lte=timezone.now()).order_by("retry_at")
 
     @classmethod
-    def claim_for_worker(cls, obj: "BaseModelWithStateMachine", lock_seconds: int = 60) -> bool:
-        """
-        Atomically claim a due object for processing using retry_at as the lock.
-
-        Correct lifecycle for any state-machine-driven work item:
-        1. Queue the item by setting retry_at <= now
-        2. Exactly one owner claims it by moving retry_at into the future
-        3. Only that owner may call .sm.tick() and perform side effects
-        4. State-machine callbacks update retry_at again when the work completes,
-           backs off, or is re-queued
-
-        The critical rule is that future retry_at values are already owned.
-        Callers must never "steal" those future timestamps and start another
-        copy of the same work. That is what prevents duplicate installs, hook
-        runs, and other concurrent side effects.
-
-        Returns True if successfully claimed, False if another worker got it
-        first or the object is not currently due.
-        """
+    def claim_for_worker(cls, obj: "ModelWithQueue", lock_seconds: int = 60) -> bool:
         now = timezone.now()
         lock_until = now + timedelta(seconds=lock_seconds)
-        updated = cls.objects.filter(
-            pk=obj.pk,
-            retry_at=obj.RETRY_AT,
-            retry_at__lte=now,
-        ).update(
+        updated = cls.objects.filter(pk=obj.pk, retry_at=obj.retry_at, retry_at__lte=now).update(
             retry_at=lock_until,
             modified_at=now,
         )
         if updated == 1:
-            obj.RETRY_AT = lock_until
+            obj.retry_at = lock_until
             cast(Any, obj).modified_at = now
         return updated == 1
 
     def claim_processing_lock(self, lock_seconds: int = 60) -> bool:
-        """
-        Claim this model instance immediately before executing one state-machine tick.
-
-        This helper is the safe entrypoint for any direct state-machine driver
-        (workers, synchronous crawl dependency installers, one-off CLI helpers).
-        Calling `.sm.tick()` without claiming first turns retry_at into "just a
-        schedule" instead of the ownership lock it is meant to be.
-
-        Returns True only for the caller that successfully moved retry_at into
-        the future. False means another process already owns the work item or it
-        is not currently due.
-        """
-        if self.STATE in self.FINAL_STATES:
+        if self.status in self.FINAL_STATES or self.retry_at is None:
             return False
-        if self.RETRY_AT is None:
-            return False
-
-        claimed = type(self).claim_for_worker(self, lock_seconds=lock_seconds)
-        return claimed
-
-    def tick_claimed(self, lock_seconds: int = 60) -> bool:
-        """
-        Claim ownership via retry_at and then execute exactly one `.sm.tick()`.
-
-        Future maintainers should prefer this helper over calling `.sm.tick()`
-        directly whenever there is any chance another process could see the same
-        queued row. If this method returns False, someone else already owns the
-        work and the caller must not run side effects for it.
-        """
-        if not self.claim_processing_lock(lock_seconds=lock_seconds):
-            return False
-
-        cast(ModelStateMachine, self.sm).tick()
-        self.refresh_from_db()
-        return True
-
-    @classproperty
-    def ACTIVE_STATE(cls) -> str:
-        return cls._state_to_str(cls.active_state)
-
-    @classproperty
-    def INITIAL_STATE(cls) -> str:
-        initial_state = cls.StateMachineClass.initial_state
-        if initial_state is None:
-            raise ValueError("StateMachineClass.initial_state must not be None")
-        return cls._state_to_str(initial_state)
-
-    @classproperty
-    def FINAL_STATES(cls) -> list[str]:
-        return [cls._state_to_str(state) for state in cls.StateMachineClass.final_states]
-
-    @classproperty
-    def FINAL_OR_ACTIVE_STATES(cls) -> list[str]:
-        return [*cls.FINAL_STATES, cls.ACTIVE_STATE]
+        return type(self).claim_for_worker(self, lock_seconds=lock_seconds)
 
     @classmethod
     def extend_choices(cls, base_choices: type[models.TextChoices]):
-        """
-        Decorator to extend the base choices with extra choices, e.g.:
-
-        class MyModel(ModelWithStateMachine):
-
-            @ModelWithStateMachine.extend_choices(ModelWithStateMachine.StatusChoices)
-            class StatusChoices(models.TextChoices):
-                SUCCEEDED = 'succeeded'
-                FAILED = 'failed'
-                SKIPPED = 'skipped'
-        """
-        assert issubclass(base_choices, models.TextChoices), (
-            f"@extend_choices(base_choices) must be a TextChoices class, not {base_choices.__name__}"
-        )
+        assert issubclass(base_choices, models.TextChoices)
 
         def wrapper(extra_choices: type[models.TextChoices]) -> type[models.TextChoices]:
-            joined = {}
-            for item in base_choices.choices:
-                joined[item[0]] = item[1]
-            for item in extra_choices.choices:
-                joined[item[0]] = item[1]
-            joined_choices = models.TextChoices("StatusChoices", joined)
-            assert isinstance(joined_choices, type)
-            return joined_choices
+            joined = {value: label for value, label in (*base_choices.choices, *extra_choices.choices)}
+            choices = models.TextChoices("StatusChoices", joined)
+            assert isinstance(choices, type)
+            return choices
 
         return wrapper
 
     @classmethod
-    def StatusField(cls, **kwargs) -> models.CharField:
-        """
-        Used on subclasses to extend/modify the status field with updated kwargs. e.g.:
-
-        class MyModel(ModelWithStateMachine):
-            class StatusChoices(ModelWithStateMachine.StatusChoices):
-                QUEUED = 'queued', 'Queued'
-                STARTED = 'started', 'Started'
-                SEALED = 'sealed', 'Sealed'
-                BACKOFF = 'backoff', 'Backoff'
-                FAILED = 'failed', 'Failed'
-                SKIPPED = 'skipped', 'Skipped'
-
-            status = ModelWithStateMachine.StatusField(choices=StatusChoices.choices, default=StatusChoices.QUEUED)
-        """
-        default_kwargs = default_status_field.deconstruct()[3]
-        updated_kwargs = {**default_kwargs, **kwargs}
-        field = models.CharField(**updated_kwargs)
-        field._is_state_field = True  # type: ignore
-        return field
+    def StatusField(cls, **kwargs: Any) -> models.CharField:
+        return models.CharField(**{**default_status_field.deconstruct()[3], **kwargs})
 
     @classmethod
-    def RetryAtField(cls, **kwargs) -> models.DateTimeField:
-        """
-        Used on subclasses to extend/modify the retry_at field with updated kwargs. e.g.:
-
-        class MyModel(ModelWithStateMachine):
-            retry_at = ModelWithStateMachine.RetryAtField(editable=False)
-        """
-        default_kwargs = default_retry_at_field.deconstruct()[3]
-        updated_kwargs = {**default_kwargs, **kwargs}
-        field = models.DateTimeField(**updated_kwargs)
-        field._is_retry_at_field = True  # type: ignore
-        return field
-
-    @classproperty
-    def StateMachineClass(cls) -> type[StateMachine]:
-        """Get the StateMachine class for the given django Model."""
-
-        model_state_machine_name = cls.state_machine_name
-        if model_state_machine_name:
-            StateMachineCls = registry.get_machine_cls(model_state_machine_name)
-            assert issubclass(StateMachineCls, StateMachine)
-            return StateMachineCls
-        raise NotImplementedError("ActorType must define .state_machine_name that points to a valid StateMachine")
-
-
-class ModelWithStateMachine(BaseModelWithStateMachine):
-    StatusChoices = DefaultStatusChoices
-
-    status: models.CharField = BaseModelWithStateMachine.StatusField()
-    retry_at: models.DateTimeField = BaseModelWithStateMachine.RetryAtField()
-
-    state_machine_name: str | None  # e.g. 'core.models.ArchiveResultMachine'
-    state_field_name: str = "status"
-    state_machine_attr: str = "sm"
-    bind_events_as_methods: bool = False
-
-    active_state = StatusChoices.STARTED
-    retry_at_field_name: str = "retry_at"
-
-    class Meta(BaseModelWithStateMachine.Meta):
-        abstract = True
-
-
-class BaseStateMachine(StateMachine):
-    """
-    Base class for all ArchiveBox state machines.
-
-    Eliminates boilerplate __init__, __repr__, __str__ methods that were
-    duplicated across all 4 state machines (Snapshot, ArchiveResult, Crawl, Binary).
-
-    Subclasses must set model_attr_name to specify the attribute name
-    (e.g., 'snapshot', 'archiveresult', 'crawl', 'binary').
-
-    Example usage:
-        class SnapshotMachine(BaseStateMachine):
-            model_attr_name = 'snapshot'
-
-            # States and transitions...
-            queued = State(value=Snapshot.StatusChoices.QUEUED, initial=True)
-            # ...
-
-    The model instance is accessible via self.{model_attr_name}
-    (e.g., self.snapshot, self.archiveresult, etc.)
-    """
-
-    model_attr_name: str = "obj"  # Override in subclasses
-
-    def __init__(self, obj, *args, **kwargs):
-        setattr(self, self.model_attr_name, obj)
-        super().__init__(obj, *args, **kwargs)
-
-    def _register_callbacks(self, listeners: list[object]):
-        """Register transition callbacks without scanning the Django model.
-
-        python-statemachine normally treats the wrapped model as a callback
-        listener.  That is useful when transition specs point at methods on the
-        domain object, but ArchiveBox keeps all transition guards/actions on the
-        machine classes themselves (`SnapshotMachine.can_start`,
-        `CrawlMachine.enter_sealed`, etc.).  Scanning the Django model therefore
-        only adds work: `dir(model)` is large, callback resolution walks that
-        attribute set for every state/transition, and the cost lands on every
-        `.sm` construction.
-
-        Keep support for explicit external listeners, but do not register
-        `self.model` as an implicit listener.  If a future machine wants model
-        methods as callbacks, pass that model explicitly as a listener at the
-        callsite so the cost is local and visible.
-        """
-        self._listeners.update({id(listener): listener for listener in listeners})
-        callbacks = Listeners.from_listeners(
-            (
-                Listener.from_obj(self, skip_attrs=self._protected_attrs),
-                *(Listener.from_obj(listener) for listener in listeners),
-            ),
-        )
-        registry = self._callbacks
-        callbacks.resolve(self._specs, registry=registry, allowed_references=SPECS_ALL)
-
-        check_callbacks = self._callbacks.check
-        for visited in iterate_states_and_transitions(self.states):
-            callbacks.resolve(visited._specs, registry=registry, allowed_references=SPECS_ALL)
-            check_callbacks(visited._specs)
-
-        self._callbacks.async_or_sync()
-
-    def __repr__(self) -> str:
-        obj = getattr(self, self.model_attr_name)
-        return f"{self.__class__.__name__}[{obj.id}]"
-
-    def __str__(self) -> str:
-        return self.__repr__()
+    def RetryAtField(cls, **kwargs: Any) -> models.DateTimeField:
+        return models.DateTimeField(**{**default_retry_at_field.deconstruct()[3], **kwargs})

bin/collect_ui_screenshots.sh

@@ -14,6 +14,7 @@ MAX_VIEWS="${UI_SCREENSHOT_MAX_VIEWS:-0}"
 USERNAME="archivebox-screenshots-$$"
 PASSWORD="archivebox-screenshots-$$"
 SERVER_PID=""
+SETUP_SERVER_PID=""
 ARCHIVE_PID=""
 CREATED_TEMP_USER=0
 CREATE_API_TOKEN=0
@@ -35,6 +36,28 @@ stop_background_runner() {
 }
 
 cleanup() {
+    if [[ -n "$SETUP_SERVER_PID" ]]; then
+        setup_server_children="$(pgrep -P "$SETUP_SERVER_PID" 2>/dev/null || true)"
+        for child_pid in $setup_server_children; do
+            kill "$child_pid" 2>/dev/null || true
+        done
+        kill "$SETUP_SERVER_PID" 2>/dev/null || true
+        for _attempt in $(seq 1 50); do
+            setup_server_running=0
+            kill -0 "$SETUP_SERVER_PID" 2>/dev/null && setup_server_running=1
+            for child_pid in $setup_server_children; do
+                kill -0 "$child_pid" 2>/dev/null && setup_server_running=1
+            done
+            [[ "$setup_server_running" == "0" ]] && break
+            sleep 0.1
+        done
+        for child_pid in $setup_server_children; do
+            kill -KILL "$child_pid" 2>/dev/null || true
+        done
+        kill -KILL "$SETUP_SERVER_PID" 2>/dev/null || true
+        wait "$SETUP_SERVER_PID" 2>/dev/null || true
+    fi
+
     if [[ -n "$ARCHIVE_PID" ]] && kill -0 "$ARCHIVE_PID" 2>/dev/null; then
         archive_children="$(pgrep -P "$ARCHIVE_PID" 2>/dev/null || true)"
         for child_pid in $archive_children; do
@@ -213,6 +236,7 @@ if [[ ! -x "$SCREENSHOT_CHROME_BINARY" ]]; then
 fi
 
 VIEWS=(
+    "First-time setup wizard|setup-wizard://admin/|/admin/|archivebox/templates/core/setup_wizard.html|setup-wizard"
     "Login|$ADMIN_BASE_URL/admin/login/|/admin/login/|archivebox/templates/admin/login.html"
     "Public snapshot list|$PUBLIC_BASE_URL/public/|/public/|archivebox/core/views.py"
 )
@@ -223,6 +247,11 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
     IFS='|' read -r name url expected_path source capture_mode <<<"$view"
     capture_index=$((capture_index + 1))
     slug="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//; s/-$//')"
+    expected_plugin=""
+    if [[ "$name" == "Snapshot View ("*")" && "$capture_mode" != "live-progress" && "$capture_mode" != "snapshot-collapsed" ]]; then
+        expected_plugin="${name#Snapshot View (}"
+        expected_plugin="${expected_plugin%)}"
+    fi
 
     echo "[*] $name: $url"
     if [[ "$capture_mode" == "snapshot-collapsed" ]]; then
@@ -262,7 +291,79 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
         fi
 
         screenshot_path=""
-        if [[ "$capture_mode" == "live-progress" ]]; then
+        if [[ "$capture_mode" == "setup-wizard" ]]; then
+            screenshot_path="$capture_dir/screenshot.png"
+            if [[ "$profile" == "desktop" ]]; then
+                SETUP_DATA_DIR="$CAPTURE_ROOT/setup-wizard-data"
+                SETUP_PORT="$(uv run --no-cache --project "$REPO_DIR" python - <<'PY'
+import socket
+
+with socket.socket() as sock:
+    sock.bind(("127.0.0.1", 0))
+    print(sock.getsockname()[1])
+PY
+)"
+                SETUP_BASE_URL="http://archivebox.localhost:$SETUP_PORT"
+                SETUP_HOST_RESOLVER_RULES="MAP archivebox.localhost 127.0.0.1"
+                mkdir -p "$SETUP_DATA_DIR" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile"
+                (
+                    cd "$SETUP_DATA_DIR"
+                    BASE_URL= uv run --no-cache --project "$REPO_DIR" archivebox init --quick
+                    UI_SCREENSHOT_USERNAME="$USERNAME" UI_SCREENSHOT_PASSWORD="$PASSWORD" \
+                        BASE_URL= uv run --no-cache --project "$REPO_DIR" archivebox manage shell --no-imports -c \
+                        'import os; from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser(username=os.environ["UI_SCREENSHOT_USERNAME"], password=os.environ["UI_SCREENSHOT_PASSWORD"])'
+                )
+                (
+                    cd "$SETUP_DATA_DIR"
+                    BASE_URL= UI_SCREENSHOT_HIDE_HIGH_LOAD_WARNING=1 \
+                        exec uv run --no-cache --project "$REPO_DIR" archivebox server "127.0.0.1:$SETUP_PORT"
+                ) >"$SETUP_DATA_DIR/ui-screenshot-setup-wizard-server.log" 2>&1 &
+                SETUP_SERVER_PID=$!
+                setup_ready=0
+                for _setup_attempt in $(seq 1 60); do
+                    if ! kill -0 "$SETUP_SERVER_PID" 2>/dev/null; then
+                        echo "[!] Setup-wizard ArchiveBox exited before becoming ready" >&2
+                        tail -100 "$SETUP_DATA_DIR/ui-screenshot-setup-wizard-server.log" >&2
+                        exit 1
+                    fi
+                    if curl --fail --silent --show-error --resolve "archivebox.localhost:$SETUP_PORT:127.0.0.1" "$SETUP_BASE_URL/admin/login/" >/dev/null; then
+                        setup_ready=1
+                        break
+                    fi
+                    sleep 1
+                done
+                if [[ "$setup_ready" != "1" ]]; then
+                    echo "[!] Setup-wizard ArchiveBox did not become ready within 60 seconds" >&2
+                    tail -100 "$SETUP_DATA_DIR/ui-screenshot-setup-wizard-server.log" >&2
+                    exit 1
+                fi
+                setup_variants="$(printf \
+                    '[{"path":"%s","width":1600,"height":1000},{"path":"%s","width":1024,"height":1366},{"path":"%s","width":390,"height":844}]' \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop/screenshot.png" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet/screenshot.png" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile/screenshot.png")"
+                NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
+                    CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
+                    SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/setup-wizard/chrome_profile" \
+                    SCREENSHOT_LOGIN_USERNAME="$USERNAME" \
+                    SCREENSHOT_LOGIN_PASSWORD="$PASSWORD" \
+                    SCREENSHOT_WIDTH=1600 \
+                    SCREENSHOT_HEIGHT=1000 \
+                    SCREENSHOT_VARIANTS_JSON="$setup_variants" \
+                    SCREENSHOT_WAIT_SELECTOR="#archivebox-setup-wizard" \
+                    SCREENSHOT_HOST_RESOLVER_RULES="$SETUP_HOST_RESOLVER_RULES" \
+                    node "$REPO_DIR/bin/take_screenshot.js" "$SETUP_BASE_URL/admin/login/?next=/admin/" "$screenshot_path" >"$capture_dir/report.json"
+                url="$SETUP_BASE_URL/admin/"
+                view_timing_report="$capture_dir/report.json"
+                timing_report_path="$view_timing_report"
+                kill "$SETUP_SERVER_PID" 2>/dev/null || true
+                wait "$SETUP_SERVER_PID" 2>/dev/null || true
+                SETUP_SERVER_PID=""
+            fi
+        elif [[ "$capture_mode" == "live-progress" ]]; then
             screenshot_path="$capture_dir/screenshot.png"
             if [[ "$profile" == "desktop" ]]; then
                 mkdir -p \
@@ -286,6 +387,32 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
                 view_timing_report="$capture_dir/report.json"
                 timing_report_path="$view_timing_report"
             fi
+        elif [[ -n "$expected_plugin" && "$capture_mode" != wait-replay:* ]]; then
+            screenshot_path="$capture_dir/screenshot.png"
+            if [[ "$profile" == "desktop" ]]; then
+                mkdir -p \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile"
+                output_variants="$(printf \
+                    '[{"path":"%s","width":1600,"height":1000},{"path":"%s","width":1024,"height":1366},{"path":"%s","width":390,"height":844}]' \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop/screenshot.png" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet/screenshot.png" \
+                    "$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile/screenshot.png")"
+                NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
+                    CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
+                    SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
+                    SCREENSHOT_WIDTH=1600 \
+                    SCREENSHOT_HEIGHT=1000 \
+                    SCREENSHOT_VARIANTS_JSON="$output_variants" \
+                    SCREENSHOT_COLLAPSE_FILTERS=1 \
+                    SCREENSHOT_EXPECT_PLUGIN="$expected_plugin" \
+                    node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
+                view_timing_report="$capture_dir/report.json"
+                timing_report_path="$view_timing_report"
+                uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
+                    "$capture_dir/report.json" "$expected_path"
+            fi
         elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then
             capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log"
             if ! env "${capture_env[@]}" uv run --no-cache --project "$REPO_DIR" abx-dl dl \
@@ -323,6 +450,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
                     SCREENSHOT_HEIGHT=1366 \
                     SCREENSHOT_VARIANTS_JSON="$responsive_variants" \
                     SCREENSHOT_COLLAPSE_FILTERS=1 \
+                    SCREENSHOT_EXPECT_PLUGIN="$expected_plugin" \
                     node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
                 view_timing_report="$capture_dir/report.json"
                 timing_report_path="$view_timing_report"
@@ -346,7 +474,7 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
 
     # Capture the public views before login because the real admin-login hint
     # intentionally redirects authenticated personas away from /public/.
-    if [[ "$capture_index" == "2" ]]; then
+    if [[ "$name" == "Public snapshot list" ]]; then
         echo "[*] Logging in through the real $ACTIVE_PERSONA browser persona"
         NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
             CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \

bin/generate_ui_screenshot_gallery.py

@@ -20,6 +20,7 @@ CAPTURE_PROFILES = {
 }
 
 REQUIRED_VIEW_NAMES = {
+    "First-time setup wizard",
     "Login",
     "Public snapshot list",
     "Add URLs",

bin/take_screenshot.js

@@ -207,10 +207,14 @@ async function main() {
       await page.waitForFunction((pluginName) => {
         const selectedCard = document.querySelector('.thumb-card.selected-card[data-plugin-name]');
         const frame = document.querySelector('#main-frame');
+        const frameRect = frame?.getBoundingClientRect();
         return selectedCard?.dataset.pluginName?.toLowerCase() === pluginName
           && frame
           && frame.getAttribute('src')
-          && frame.getAttribute('src') !== 'about:blank';
+          && frame.getAttribute('src') !== 'about:blank'
+          && frameRect
+          && frameRect.width >= Math.min(window.innerWidth * 0.9, window.innerWidth - 8)
+          && frameRect.height >= Math.min(window.innerHeight * 0.35, 320);
       }, { timeout: 45000 }, expectedPlugin);
       await page.waitForFunction(() => {
         const frames = [document.querySelector('#main-frame'), document.querySelector('.thumb-card.selected-card iframe')]

docs/ArchiveBox-Architecture-Diagrams.md

@@ -4,8 +4,8 @@ This page is a map of the current execution and persistence paths. The implement
 
 - `archivebox/cli/` for CLI entry points
 - `archivebox/services/runner.py` for crawl and snapshot execution
-- `archivebox/crawls/models.py` for the `Crawl` model and state machine
-- `archivebox/core/models.py` for `Snapshot`, `ArchiveResult`, and the `Snapshot` state machine
+- `archivebox/crawls/models.py` for the `Crawl` model and its atomic queue transitions
+- `archivebox/core/models.py` for `Snapshot`, `ArchiveResult`, and Snapshot queue transitions
 - `archivebox/services/` for bus event projectors
 - `abxpkg` and `abx-plugins` for binary resolution and plugin hooks
 
@@ -57,44 +57,49 @@ flowchart LR
 
 The database is the source of truth for model state. Snapshot directories contain captured artifacts and rendered metadata. Older collections may also contain legacy timestamp-named snapshot directories.
 
-## `Crawl` State Machine
+## `Crawl` Queue Lifecycle
 
-Implemented by `Crawl` and `CrawlMachine` in `archivebox/crawls/models.py`.
+Implemented directly by `Crawl` in `archivebox/crawls/models.py`. The database
+row is the durable state; the runner claims `retry_at` with a conditional update
+before it performs side effects, then calls the model's explicit lifecycle
+methods. There is deliberately no second in-memory state machine that can drift
+from the row owned by another process.
 
 ```mermaid
 stateDiagram-v2
     [*] --> QUEUED
-    QUEUED --> STARTED: tick and valid URLs
-    QUEUED --> QUEUED: tick and not ready
+    QUEUED --> STARTED: runner claim and valid URLs
+    QUEUED --> QUEUED: claimed but not ready
     QUEUED --> SEALED: all existing snapshots finished
     STARTED --> SEALED: all snapshots finished
     QUEUED --> PAUSED: pause requested
     STARTED --> PAUSED: pause requested
     PAUSED --> QUEUED: resume requested
-    PAUSED --> PAUSED: tick
+    PAUSED --> PAUSED: not runnable
     QUEUED --> SEALED: explicit seal
     STARTED --> SEALED: explicit seal
     PAUSED --> SEALED: explicit seal
     SEALED --> [*]
 ```
 
-A crawl owns a set of snapshots. Entering `STARTED` creates or discovers those snapshots; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue.
+A crawl owns a set of snapshots. The runner creates or discovers those snapshots and projects crawl events while the row is `STARTED`; sealing waits for their normal lifecycle to finish. Pausing also schedules child snapshots to pause, and resuming returns the crawl to the runnable queue. The `archivebox://update` sentinel remains control-plane work: the runner invokes database maintenance directly and never turns it into a Snapshot.
 
-## `Snapshot` State Machine
+## `Snapshot` Queue Lifecycle
 
-Implemented by `Snapshot` and `SnapshotMachine` in `archivebox/core/models.py`.
+Implemented directly by `Snapshot` in `archivebox/core/models.py`, using the
+same conditional `retry_at` claim protocol as `Crawl`.
 
 ```mermaid
 stateDiagram-v2
     [*] --> QUEUED
-    QUEUED --> STARTED: tick and URL is ready
-    QUEUED --> QUEUED: tick and not ready
+    QUEUED --> STARTED: runner claim and URL is ready
+    QUEUED --> QUEUED: claimed but not ready
     QUEUED --> SEALED: all existing results finished
     STARTED --> SEALED: all hook results finished
     QUEUED --> PAUSED: pause requested
     STARTED --> PAUSED: pause requested
     PAUSED --> QUEUED: resume requested
-    PAUSED --> PAUSED: tick
+    PAUSED --> PAUSED: not runnable
     QUEUED --> SEALED: explicit seal
     STARTED --> SEALED: explicit seal
     PAUSED --> SEALED: explicit seal
@@ -105,7 +110,7 @@ The runner creates one queued `ArchiveResult` per selected hook, executes those
 
 ## `ArchiveResult` Projection
 
-`ArchiveResult` is not driven by a separate Python state machine. The runner creates queued rows, and `ArchiveResultService` projects `ArchiveResultEvent` and `ProcessCompletedEvent` data into them.
+`ArchiveResult` is not driven by a separate in-memory state machine. The runner creates queued rows, and `ArchiveResultService` projects `ArchiveResultEvent` and `ProcessCompletedEvent` data into them.
 
 ```mermaid
 flowchart LR

docs/apidocs/archivebox/archivebox.core.models.md

@@ -33,16 +33,28 @@
     ```
 * - {py:obj}`Snapshot <archivebox.core.models.Snapshot>`
   -
-* - {py:obj}`SnapshotMachine <archivebox.core.models.SnapshotMachine>`
-  - ```{autodoc2-docstring} archivebox.core.models.SnapshotMachine
-    :summary:
-    ```
 * - {py:obj}`ArchiveResult <archivebox.core.models.ArchiveResult>`
   -
 ````
 
 ### API
 
+````{py:exception} SnapshotMigrationError()
+:canonical: archivebox.core.models.SnapshotMigrationError
+
+Bases: {py:obj}`RuntimeError`
+
+```{autodoc2-docstring} archivebox.core.models.SnapshotMigrationError
+```
+
+```{rubric} Initialization
+```
+
+```{autodoc2-docstring} archivebox.core.models.SnapshotMigrationError.__init__
+```
+
+````
+
 `````{py:class} UngroupedSubquery(queryset, output_field=None, **extra)
 :canonical: archivebox.core.models.UngroupedSubquery
 
@@ -280,6 +292,7 @@ Bases: {py:obj}`django.db.models.Model`
 
 ````{py:attribute} unique_together
 :canonical: archivebox.core.models.SnapshotTag.Meta.unique_together
+:type: typing.ClassVar[list[tuple[str, str]]]
 :value: >
    [('snapshot', 'tag')]
 
@@ -321,6 +334,7 @@ Bases: {py:obj}`django.db.models.QuerySet`
 
 ````{py:attribute} FILTER_TYPES
 :canonical: archivebox.core.models.SnapshotQuerySet.FILTER_TYPES
+:type: typing.ClassVar[dict[str, typing.Any]]
 :value: >
    None
 
@@ -438,7 +452,17 @@ Bases: {py:obj}`models.Manager.from_queryset`\({py:obj}`SnapshotQuerySet`\)
 ``````{py:class} Snapshot(*args, **kwargs)
 :canonical: archivebox.core.models.Snapshot
 
-Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine`
+Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithNotes`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithQueue`
+
+````{py:attribute} BROWSER_EXTENSION_UPLOAD_HOOK_NAME
+:canonical: archivebox.core.models.Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
+:value: >
+   'on_Snapshot__archivebox_browser_extension_upload'
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME
+```
+
+````
 
 ````{py:attribute} INTERNAL_INPUT_URL
 :canonical: archivebox.core.models.Snapshot.INTERNAL_INPUT_URL
@@ -651,16 +675,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar
 
 ````
 
-````{py:attribute} state_machine_name
-:canonical: archivebox.core.models.Snapshot.state_machine_name
-:value: >
-   'archivebox.core.models.SnapshotMachine'
-
-```{autodoc2-docstring} archivebox.core.models.Snapshot.state_machine_name
-```
-
-````
-
 ````{py:attribute} state_field_name
 :canonical: archivebox.core.models.Snapshot.state_field_name
 :value: >
@@ -691,6 +705,46 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar
 
 ````
 
+````{py:attribute} INITIAL_STATE
+:canonical: archivebox.core.models.Snapshot.INITIAL_STATE
+:value: >
+   None
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.INITIAL_STATE
+```
+
+````
+
+````{py:attribute} ACTIVE_STATE
+:canonical: archivebox.core.models.Snapshot.ACTIVE_STATE
+:value: >
+   None
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.ACTIVE_STATE
+```
+
+````
+
+````{py:attribute} FINAL_STATES
+:canonical: archivebox.core.models.Snapshot.FINAL_STATES
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.FINAL_STATES
+```
+
+````
+
+````{py:attribute} FINAL_OR_ACTIVE_STATES
+:canonical: archivebox.core.models.Snapshot.FINAL_OR_ACTIVE_STATES
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.FINAL_OR_ACTIVE_STATES
+```
+
+````
+
 ````{py:attribute} active_state
 :canonical: archivebox.core.models.Snapshot.active_state
 :value: >
@@ -796,7 +850,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar
 `````{py:class} Meta
 :canonical: archivebox.core.models.Snapshot.Meta
 
-Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithNotes.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta`
+Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithNotes.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithQueue.Meta`
 
 ````{py:attribute} app_label
 :canonical: archivebox.core.models.Snapshot.Meta.app_label
@@ -830,6 +884,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````{py:attribute} indexes
 :canonical: archivebox.core.models.Snapshot.Meta.indexes
+:type: typing.ClassVar[list[django.db.models.Index]]
 :value: >
    None
 
@@ -840,6 +895,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````{py:attribute} constraints
 :canonical: archivebox.core.models.Snapshot.Meta.constraints
+:type: typing.ClassVar[list[django.db.models.BaseConstraint]]
 :value: >
    None
 
@@ -946,6 +1002,30 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
+````{py:method} start_processing() -> bool
+:canonical: archivebox.core.models.Snapshot.start_processing
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.start_processing
+```
+
+````
+
+````{py:method} seal() -> bool
+:canonical: archivebox.core.models.Snapshot.seal
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.seal
+```
+
+````
+
+````{py:method} advance_lifecycle() -> bool
+:canonical: archivebox.core.models.Snapshot.advance_lifecycle
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.advance_lifecycle
+```
+
+````
+
 ````{py:method} cancel() -> None
 :canonical: archivebox.core.models.Snapshot.cancel
 
@@ -1042,6 +1122,17 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
+````{py:attribute} _FS_VERSION_MIGRATION_PATHS
+:canonical: archivebox.core.models.Snapshot._FS_VERSION_MIGRATION_PATHS
+:type: typing.ClassVar[dict[str, str]]
+:value: >
+   None
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot._FS_VERSION_MIGRATION_PATHS
+```
+
+````
+
 ````{py:property} fs_migration_needed
 :canonical: archivebox.core.models.Snapshot.fs_migration_needed
 :type: bool
@@ -1100,6 +1191,14 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
+````{py:method} hydrate_archiveresult_output_metadata(snapshot_dir: pathlib.Path | None = None) -> int
+:canonical: archivebox.core.models.Snapshot.hydrate_archiveresult_output_metadata
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.hydrate_archiveresult_output_metadata
+```
+
+````
+
 ````{py:method} _fs_migrate_legacy_to_0_9_0(source_dir: pathlib.Path | None = None, target_dir: pathlib.Path | None = None, config: ArchiveBoxBaseConfig | None = None)
 :canonical: archivebox.core.models.Snapshot._fs_migrate_legacy_to_0_9_0
 
@@ -1311,7 +1410,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} tags_str(nocache=True) -> str | None
+````{py:method} tags_str() -> str | None
 :canonical: archivebox.core.models.Snapshot.tags_str
 
 ```{autodoc2-docstring} archivebox.core.models.Snapshot.tags_str
@@ -1319,7 +1418,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} icons(path: str | None = None) -> str
+````{py:method} icons(path: str | None = None, prefix: str = '/', quote_paths: bool = False) -> str
 :canonical: archivebox.core.models.Snapshot.icons
 
 ```{autodoc2-docstring} archivebox.core.models.Snapshot.icons
@@ -1404,6 +1503,14 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
+````{py:method} remove_legacy_archive_symlink() -> None
+:canonical: archivebox.core.models.Snapshot.remove_legacy_archive_symlink
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.remove_legacy_archive_symlink
+```
+
+````
+
 ````{py:method} legacy_archive_path() -> str
 :canonical: archivebox.core.models.Snapshot.legacy_archive_path
 
@@ -1468,10 +1575,10 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} cleanup()
-:canonical: archivebox.core.models.Snapshot.cleanup
+````{py:method} finalize_output_metadata() -> None
+:canonical: archivebox.core.models.Snapshot.finalize_output_metadata
 
-```{autodoc2-docstring} archivebox.core.models.Snapshot.cleanup
+```{autodoc2-docstring} archivebox.core.models.Snapshot.finalize_output_metadata
 ```
 
 ````
@@ -1653,7 +1760,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} discover_outputs(include_filesystem_fallback: bool = True) -> list[dict]
+````{py:method} discover_outputs(include_filesystem_fallback: bool = True, archive_results: list[archivebox.core.models.Snapshot.discover_outputs.ArchiveResult] | None = None) -> list[dict]
 :canonical: archivebox.core.models.Snapshot.discover_outputs
 
 ```{autodoc2-docstring} archivebox.core.models.Snapshot.discover_outputs
@@ -1661,7 +1768,16 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} to_dict(extended: bool = False) -> dict[str, typing.Any]
+````{py:property} static_archive_path
+:canonical: archivebox.core.models.Snapshot.static_archive_path
+:type: str
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.static_archive_path
+```
+
+````
+
+````{py:method} to_dict(extended: bool = False, static_export: bool = False) -> dict[str, typing.Any]
 :canonical: archivebox.core.models.Snapshot.to_dict
 
 ```{autodoc2-docstring} archivebox.core.models.Snapshot.to_dict
@@ -1693,6 +1809,14 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
+````{py:method} get_html_details_context(request=None, *, static_export_dir: pathlib.Path | None = None) -> dict[str, typing.Any]
+:canonical: archivebox.core.models.Snapshot.get_html_details_context
+
+```{autodoc2-docstring} archivebox.core.models.Snapshot.get_html_details_context
+```
+
+````
+
 ````{py:method} write_html_details(out_dir: pathlib.Path | str | None = None) -> None
 :canonical: archivebox.core.models.Snapshot.write_html_details
 
@@ -1701,7 +1825,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} get_detail_page_auxiliary_items(outputs: list[dict] | None = None, hidden_card_plugins: set[str] | None = None) -> tuple[list[dict[str, object]], list[dict[str, object]]]
+````{py:method} get_detail_page_auxiliary_items(outputs: list[dict] | None = None, hidden_card_plugins: set[str] | None = None, archive_results: list[archivebox.core.models.Snapshot.get_detail_page_auxiliary_items.ArchiveResult] | None = None) -> tuple[list[dict[str, object]], list[dict[str, object]]]
 :canonical: archivebox.core.models.Snapshot.get_detail_page_auxiliary_items
 
 ```{autodoc2-docstring} archivebox.core.models.Snapshot.get_detail_page_auxiliary_items
@@ -1720,185 +1844,12 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ``````
 
-`````{py:class} SnapshotMachine(obj, *args, **kwargs)
-:canonical: archivebox.core.models.SnapshotMachine
-
-Bases: {py:obj}`archivebox.workers.models.BaseStateMachine`
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine
-```
-
-```{rubric} Initialization
-```
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.__init__
-```
-
-````{py:attribute} model_attr_name
-:canonical: archivebox.core.models.SnapshotMachine.model_attr_name
-:value: >
-   'snapshot'
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.model_attr_name
-```
-
-````
-
-````{py:attribute} queued
-:canonical: archivebox.core.models.SnapshotMachine.queued
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.queued
-```
-
-````
-
-````{py:attribute} started
-:canonical: archivebox.core.models.SnapshotMachine.started
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.started
-```
-
-````
-
-````{py:attribute} paused
-:canonical: archivebox.core.models.SnapshotMachine.paused
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.paused
-```
-
-````
-
-````{py:attribute} sealed
-:canonical: archivebox.core.models.SnapshotMachine.sealed
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.sealed
-```
-
-````
-
-````{py:attribute} tick
-:canonical: archivebox.core.models.SnapshotMachine.tick
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.tick
-```
-
-````
-
-````{py:attribute} seal
-:canonical: archivebox.core.models.SnapshotMachine.seal
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.seal
-```
-
-````
-
-````{py:attribute} pause_requested
-:canonical: archivebox.core.models.SnapshotMachine.pause_requested
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.pause_requested
-```
-
-````
-
-````{py:attribute} resume_requested
-:canonical: archivebox.core.models.SnapshotMachine.resume_requested
-:value: >
-   'to(...)'
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.resume_requested
-```
-
-````
-
-````{py:attribute} snapshot
-:canonical: archivebox.core.models.SnapshotMachine.snapshot
-:type: archivebox.core.models.Snapshot
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.snapshot
-```
-
-````
-
-````{py:method} can_start() -> bool
-:canonical: archivebox.core.models.SnapshotMachine.can_start
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.can_start
-```
-
-````
-
-````{py:method} is_finished() -> bool
-:canonical: archivebox.core.models.SnapshotMachine.is_finished
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.is_finished
-```
-
-````
-
-````{py:method} has_finished_archive_results() -> bool
-:canonical: archivebox.core.models.SnapshotMachine.has_finished_archive_results
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.has_finished_archive_results
-```
-
-````
-
-````{py:method} enter_queued()
-:canonical: archivebox.core.models.SnapshotMachine.enter_queued
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_queued
-```
-
-````
-
-````{py:method} enter_paused()
-:canonical: archivebox.core.models.SnapshotMachine.enter_paused
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_paused
-```
-
-````
-
-````{py:method} enter_started()
-:canonical: archivebox.core.models.SnapshotMachine.enter_started
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_started
-```
-
-````
-
-````{py:method} enter_sealed()
-:canonical: archivebox.core.models.SnapshotMachine.enter_sealed
-
-```{autodoc2-docstring} archivebox.core.models.SnapshotMachine.enter_sealed
-```
-
-````
-
-`````
-
 ``````{py:class} ArchiveResult(*args, **kwargs)
 :canonical: archivebox.core.models.ArchiveResult
 
 Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithNotes`
 
-`````{py:class} StatusChoices(*args, **kwds)
+`````{py:class} StatusChoices()
 :canonical: archivebox.core.models.ArchiveResult.StatusChoices
 
 Bases: {py:obj}`django.db.models.TextChoices`
@@ -2107,24 +2058,6 @@ Bases: {py:obj}`django.db.models.TextChoices`
 
 ````
 
-````{py:method} cached_snapshot_ids_with_majority_status(status: str | collections.abc.Iterable[str], *, timeout: int = 60) -> tuple[str, ...]
-:canonical: archivebox.core.models.ArchiveResult.cached_snapshot_ids_with_majority_status
-:classmethod:
-
-```{autodoc2-docstring} archivebox.core.models.ArchiveResult.cached_snapshot_ids_with_majority_status
-```
-
-````
-
-````{py:method} clear_majority_status_cache() -> None
-:canonical: archivebox.core.models.ArchiveResult.clear_majority_status_cache
-:classmethod:
-
-```{autodoc2-docstring} archivebox.core.models.ArchiveResult.clear_majority_status_cache
-```
-
-````
-
 ````{py:attribute} id
 :canonical: archivebox.core.models.ArchiveResult.id
 :value: >
@@ -2346,7 +2279,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 ````{py:attribute} verbose_name_plural
 :canonical: archivebox.core.models.ArchiveResult.Meta.verbose_name_plural
 :value: >
-   'Archive Results Log'
+   'Archive Results'
 
 ```{autodoc2-docstring} archivebox.core.models.ArchiveResult.Meta.verbose_name_plural
 ```
@@ -2355,6 +2288,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````{py:attribute} indexes
 :canonical: archivebox.core.models.ArchiveResult.Meta.indexes
+:type: typing.ClassVar[list[django.db.models.Index]]
 :value: >
    None
 
@@ -2365,6 +2299,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````{py:attribute} constraints
 :canonical: archivebox.core.models.ArchiveResult.Meta.constraints
+:type: typing.ClassVar[list[django.db.models.BaseConstraint]]
 :value: >
    None
 
@@ -2444,10 +2379,10 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} delete(*args, **kwargs)
-:canonical: archivebox.core.models.ArchiveResult.delete
+````{py:method} schedule_delete_cleanup(*, using: str | None = None) -> None
+:canonical: archivebox.core.models.ArchiveResult.schedule_delete_cleanup
 
-```{autodoc2-docstring} archivebox.core.models.ArchiveResult.delete
+```{autodoc2-docstring} archivebox.core.models.ArchiveResult.schedule_delete_cleanup
 ```
 
 ````
@@ -2765,14 +2700,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} update_from_output()
-:canonical: archivebox.core.models.ArchiveResult.update_from_output
-
-```{autodoc2-docstring} archivebox.core.models.ArchiveResult.update_from_output
-```
-
-````
-
 ````{py:method} _set_binary_from_cmd(cmd: list) -> None
 :canonical: archivebox.core.models.ArchiveResult._set_binary_from_cmd
 

docs/apidocs/archivebox/archivebox.crawls.models.md

@@ -19,8 +19,6 @@
   -
 * - {py:obj}`Crawl <archivebox.crawls.models.Crawl>`
   -
-* - {py:obj}`CrawlMachine <archivebox.crawls.models.CrawlMachine>`
-  -
 ````
 
 ### API
@@ -235,7 +233,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithUUID.Meta`, {py:obj}`arch
 ``````{py:class} Crawl(*args, **kwargs)
 :canonical: archivebox.crawls.models.Crawl
 
-Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine`
+Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir`, {py:obj}`archivebox.base_models.models.ModelWithConfig`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithQueue`
 
 ````{py:attribute} id
 :canonical: archivebox.crawls.models.Crawl.id
@@ -387,16 +385,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar
 
 ````
 
-````{py:attribute} state_machine_name
-:canonical: archivebox.crawls.models.Crawl.state_machine_name
-:value: >
-   'archivebox.crawls.models.CrawlMachine'
-
-```{autodoc2-docstring} archivebox.crawls.models.Crawl.state_machine_name
-```
-
-````
-
 ````{py:attribute} retry_at_field_name
 :canonical: archivebox.crawls.models.Crawl.retry_at_field_name
 :value: >
@@ -427,6 +415,46 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar
 
 ````
 
+````{py:attribute} INITIAL_STATE
+:canonical: archivebox.crawls.models.Crawl.INITIAL_STATE
+:value: >
+   None
+
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.INITIAL_STATE
+```
+
+````
+
+````{py:attribute} ACTIVE_STATE
+:canonical: archivebox.crawls.models.Crawl.ACTIVE_STATE
+:value: >
+   None
+
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.ACTIVE_STATE
+```
+
+````
+
+````{py:attribute} FINAL_STATES
+:canonical: archivebox.crawls.models.Crawl.FINAL_STATES
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.FINAL_STATES
+```
+
+````
+
+````{py:attribute} FINAL_OR_ACTIVE_STATES
+:canonical: archivebox.crawls.models.Crawl.FINAL_OR_ACTIVE_STATES
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.FINAL_OR_ACTIVE_STATES
+```
+
+````
+
 ````{py:attribute} active_state
 :canonical: archivebox.crawls.models.Crawl.active_state
 :value: >
@@ -492,7 +520,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`ar
 `````{py:class} Meta
 :canonical: archivebox.crawls.models.Crawl.Meta
 
-Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta`
+Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:obj}`archivebox.base_models.models.ModelWithOutputDir.Meta`, {py:obj}`archivebox.base_models.models.ModelWithConfig.Meta`, {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithQueue.Meta`
 
 ````{py:attribute} app_label
 :canonical: archivebox.crawls.models.Crawl.Meta.app_label
@@ -940,14 +968,6 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} run() -> Snapshot | None
-:canonical: archivebox.crawls.models.Crawl.run
-
-```{autodoc2-docstring} archivebox.crawls.models.Crawl.run
-```
-
-````
-
 ````{py:method} is_finished() -> bool
 :canonical: archivebox.crawls.models.Crawl.is_finished
 
@@ -956,176 +976,52 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`, {py:ob
 
 ````
 
-````{py:method} cleanup()
-:canonical: archivebox.crawls.models.Crawl.cleanup
-
-```{autodoc2-docstring} archivebox.crawls.models.Crawl.cleanup
-```
-
-````
-
-``````
-
-`````{py:class} CrawlMachine(obj, *args, **kwargs)
-:canonical: archivebox.crawls.models.CrawlMachine
-
-Bases: {py:obj}`archivebox.workers.models.BaseStateMachine`
-
-````{py:attribute} crawl
-:canonical: archivebox.crawls.models.CrawlMachine.crawl
-:type: archivebox.crawls.models.Crawl
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.crawl
-```
-
-````
-
-````{py:attribute} model_attr_name
-:canonical: archivebox.crawls.models.CrawlMachine.model_attr_name
-:value: >
-   'crawl'
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.model_attr_name
-```
-
-````
-
-````{py:attribute} queued
-:canonical: archivebox.crawls.models.CrawlMachine.queued
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.queued
-```
-
-````
-
-````{py:attribute} started
-:canonical: archivebox.crawls.models.CrawlMachine.started
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.started
-```
-
-````
-
-````{py:attribute} paused
-:canonical: archivebox.crawls.models.CrawlMachine.paused
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.paused
-```
-
-````
-
-````{py:attribute} sealed
-:canonical: archivebox.crawls.models.CrawlMachine.sealed
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.sealed
-```
-
-````
-
-````{py:attribute} tick
-:canonical: archivebox.crawls.models.CrawlMachine.tick
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.tick
-```
-
-````
-
-````{py:attribute} seal
-:canonical: archivebox.crawls.models.CrawlMachine.seal
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.seal
-```
-
-````
-
-````{py:attribute} pause_requested
-:canonical: archivebox.crawls.models.CrawlMachine.pause_requested
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.pause_requested
-```
-
-````
-
-````{py:attribute} resume_requested
-:canonical: archivebox.crawls.models.CrawlMachine.resume_requested
-:value: >
-   'to(...)'
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.resume_requested
-```
-
-````
-
 ````{py:method} can_start() -> bool
-:canonical: archivebox.crawls.models.CrawlMachine.can_start
+:canonical: archivebox.crawls.models.Crawl.can_start
 
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.can_start
-```
-
-````
-
-````{py:method} is_finished() -> bool
-:canonical: archivebox.crawls.models.CrawlMachine.is_finished
-
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.is_finished
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.can_start
 ```
 
 ````
 
 ````{py:method} has_finished_snapshots() -> bool
-:canonical: archivebox.crawls.models.CrawlMachine.has_finished_snapshots
+:canonical: archivebox.crawls.models.Crawl.has_finished_snapshots
 
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.has_finished_snapshots
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.has_finished_snapshots
 ```
 
 ````
 
-````{py:method} enter_queued()
-:canonical: archivebox.crawls.models.CrawlMachine.enter_queued
+````{py:method} mark_started() -> bool
+:canonical: archivebox.crawls.models.Crawl.mark_started
 
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_queued
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.mark_started
 ```
 
 ````
 
-````{py:method} enter_started()
-:canonical: archivebox.crawls.models.CrawlMachine.enter_started
+````{py:method} seal() -> bool
+:canonical: archivebox.crawls.models.Crawl.seal
 
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_started
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.seal
 ```
 
 ````
 
-````{py:method} enter_paused()
-:canonical: archivebox.crawls.models.CrawlMachine.enter_paused
+````{py:method} advance_lifecycle() -> bool
+:canonical: archivebox.crawls.models.Crawl.advance_lifecycle
 
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_paused
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.advance_lifecycle
 ```
 
 ````
 
-````{py:method} enter_sealed()
-:canonical: archivebox.crawls.models.CrawlMachine.enter_sealed
+````{py:method} cleanup_runtime() -> None
+:canonical: archivebox.crawls.models.Crawl.cleanup_runtime
 
-```{autodoc2-docstring} archivebox.crawls.models.CrawlMachine.enter_sealed
+```{autodoc2-docstring} archivebox.crawls.models.Crawl.cleanup_runtime
 ```
 
 ````
 
-`````
+``````

docs/apidocs/archivebox/archivebox.machine.models.md

@@ -43,14 +43,6 @@
   - ```{autodoc2-docstring} archivebox.machine.models.Process
     :summary:
     ```
-* - {py:obj}`BinaryMachine <archivebox.machine.models.BinaryMachine>`
-  - ```{autodoc2-docstring} archivebox.machine.models.BinaryMachine
-    :summary:
-    ```
-* - {py:obj}`ProcessMachine <archivebox.machine.models.ProcessMachine>`
-  - ```{autodoc2-docstring} archivebox.machine.models.ProcessMachine
-    :summary:
-    ```
 ````
 
 ### Functions
@@ -59,6 +51,10 @@
 :class: autosummary longtable
 :align: left
 
+* - {py:obj}`get_current_pid_namespace <archivebox.machine.models.get_current_pid_namespace>`
+  - ```{autodoc2-docstring} archivebox.machine.models.get_current_pid_namespace
+    :summary:
+    ```
 * - {py:obj}`_default_exit_code_for_unowned_process <archivebox.machine.models._default_exit_code_for_unowned_process>`
   - ```{autodoc2-docstring} archivebox.machine.models._default_exit_code_for_unowned_process
     :summary:
@@ -135,6 +131,10 @@
   - ```{autodoc2-docstring} archivebox.machine.models.START_TIME_TOLERANCE
     :summary:
     ```
+* - {py:obj}`PROCESS_PID_NAMESPACE_KEY <archivebox.machine.models.PROCESS_PID_NAMESPACE_KEY>`
+  - ```{autodoc2-docstring} archivebox.machine.models.PROCESS_PID_NAMESPACE_KEY
+    :summary:
+    ```
 ````
 
 ### API
@@ -264,6 +264,23 @@
 
 ````
 
+````{py:data} PROCESS_PID_NAMESPACE_KEY
+:canonical: archivebox.machine.models.PROCESS_PID_NAMESPACE_KEY
+:value: >
+   '_ARCHIVEBOX_PID_NAMESPACE'
+
+```{autodoc2-docstring} archivebox.machine.models.PROCESS_PID_NAMESPACE_KEY
+```
+
+````
+
+````{py:function} get_current_pid_namespace() -> str
+:canonical: archivebox.machine.models.get_current_pid_namespace
+
+```{autodoc2-docstring} archivebox.machine.models.get_current_pid_namespace
+```
+````
+
 ````{py:function} _default_exit_code_for_unowned_process(process_type: str) -> int
 :canonical: archivebox.machine.models._default_exit_code_for_unowned_process
 
@@ -853,7 +870,7 @@ Bases: {py:obj}`django.db.models.Manager`
 ``````{py:class} Binary(*args, **kwargs)
 :canonical: archivebox.machine.models.Binary
 
-Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithStateMachine`
+Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`archivebox.workers.models.ModelWithQueue`
 
 ```{autodoc2-docstring} archivebox.machine.models.Binary
 ```
@@ -864,7 +881,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats`, {py:obj}`ar
 ```{autodoc2-docstring} archivebox.machine.models.Binary.__init__
 ```
 
-`````{py:class} StatusChoices(*args, **kwds)
+`````{py:class} StatusChoices()
 :canonical: archivebox.machine.models.Binary.StatusChoices
 
 Bases: {py:obj}`django.db.models.TextChoices`
@@ -1052,13 +1069,42 @@ Bases: {py:obj}`django.db.models.TextChoices`
 
 ````
 
-````{py:attribute} state_machine_name
-:canonical: archivebox.machine.models.Binary.state_machine_name
-:type: str | None
+````{py:attribute} INITIAL_STATE
+:canonical: archivebox.machine.models.Binary.INITIAL_STATE
 :value: >
-   'archivebox.machine.models.BinaryMachine'
+   None
 
-```{autodoc2-docstring} archivebox.machine.models.Binary.state_machine_name
+```{autodoc2-docstring} archivebox.machine.models.Binary.INITIAL_STATE
+```
+
+````
+
+````{py:attribute} ACTIVE_STATE
+:canonical: archivebox.machine.models.Binary.ACTIVE_STATE
+:value: >
+   None
+
+```{autodoc2-docstring} archivebox.machine.models.Binary.ACTIVE_STATE
+```
+
+````
+
+````{py:attribute} FINAL_STATES
+:canonical: archivebox.machine.models.Binary.FINAL_STATES
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.machine.models.Binary.FINAL_STATES
+```
+
+````
+
+````{py:attribute} FINAL_OR_ACTIVE_STATES
+:canonical: archivebox.machine.models.Binary.FINAL_OR_ACTIVE_STATES
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.machine.models.Binary.FINAL_OR_ACTIVE_STATES
 ```
 
 ````
@@ -1097,7 +1143,7 @@ Bases: {py:obj}`django.db.models.TextChoices`
 `````{py:class} Meta
 :canonical: archivebox.machine.models.Binary.Meta
 
-Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithStateMachine.Meta`
+Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:obj}`archivebox.workers.models.ModelWithQueue.Meta`
 
 ````{py:attribute} app_label
 :canonical: archivebox.machine.models.Binary.Meta.app_label
@@ -1155,6 +1201,15 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:ob
 
 ````
 
+````{py:property} can_install
+:canonical: archivebox.machine.models.Binary.can_install
+:type: bool
+
+```{autodoc2-docstring} archivebox.machine.models.Binary.can_install
+```
+
+````
+
 ````{py:method} binary_info() -> dict
 :canonical: archivebox.machine.models.Binary.binary_info
 
@@ -1205,6 +1260,30 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithHealthStats.Meta`, {py:ob
 
 ````
 
+````{py:method} install() -> bool
+:canonical: archivebox.machine.models.Binary.install
+
+```{autodoc2-docstring} archivebox.machine.models.Binary.install
+```
+
+````
+
+````{py:method} advance_lifecycle() -> bool
+:canonical: archivebox.machine.models.Binary.advance_lifecycle
+
+```{autodoc2-docstring} archivebox.machine.models.Binary.advance_lifecycle
+```
+
+````
+
+````{py:method} install_claimed(*, lock_seconds: int = 600) -> bool
+:canonical: archivebox.machine.models.Binary.install_claimed
+
+```{autodoc2-docstring} archivebox.machine.models.Binary.install_claimed
+```
+
+````
+
 ````{py:method} cleanup()
 :canonical: archivebox.machine.models.Binary.cleanup
 
@@ -1279,7 +1358,7 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter`, {py:obj}`dj
 ```{autodoc2-docstring} archivebox.machine.models.Process.__init__
 ```
 
-`````{py:class} StatusChoices(*args, **kwds)
+`````{py:class} StatusChoices()
 :canonical: archivebox.machine.models.Process.StatusChoices
 
 Bases: {py:obj}`django.db.models.TextChoices`
@@ -1316,7 +1395,7 @@ Bases: {py:obj}`django.db.models.TextChoices`
 
 `````
 
-`````{py:class} TypeChoices(*args, **kwds)
+`````{py:class} TypeChoices()
 :canonical: archivebox.machine.models.Process.TypeChoices
 
 Bases: {py:obj}`django.db.models.TextChoices`
@@ -1698,17 +1777,6 @@ Bases: {py:obj}`django.db.models.TextChoices`
 
 ````
 
-````{py:attribute} state_machine_name
-:canonical: archivebox.machine.models.Process.state_machine_name
-:type: str
-:value: >
-   'archivebox.machine.models.ProcessMachine'
-
-```{autodoc2-docstring} archivebox.machine.models.Process.state_machine_name
-```
-
-````
-
 ````{py:attribute} delete_after_final_statuses
 :canonical: archivebox.machine.models.Process.delete_after_final_statuses
 :value: >
@@ -1997,6 +2065,15 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`
 
 ````
 
+````{py:property} shares_pid_namespace
+:canonical: archivebox.machine.models.Process.shares_pid_namespace
+:type: bool
+
+```{autodoc2-docstring} archivebox.machine.models.Process.shares_pid_namespace
+```
+
+````
+
 ````{py:property} proc
 :canonical: archivebox.machine.models.Process.proc
 :type: psutil.Process | None
@@ -2242,239 +2319,3 @@ Bases: {py:obj}`archivebox.base_models.models.ModelWithDeleteAfter.Meta`
 ````
 
 ``````
-
-`````{py:class} BinaryMachine(obj, *args, **kwargs)
-:canonical: archivebox.machine.models.BinaryMachine
-
-Bases: {py:obj}`archivebox.workers.models.BaseStateMachine`
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine
-```
-
-```{rubric} Initialization
-```
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.__init__
-```
-
-````{py:attribute} model_attr_name
-:canonical: archivebox.machine.models.BinaryMachine.model_attr_name
-:value: >
-   'binary'
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.model_attr_name
-```
-
-````
-
-````{py:attribute} binary
-:canonical: archivebox.machine.models.BinaryMachine.binary
-:type: archivebox.machine.models.Binary
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.binary
-```
-
-````
-
-````{py:attribute} queued
-:canonical: archivebox.machine.models.BinaryMachine.queued
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.queued
-```
-
-````
-
-````{py:attribute} installed
-:canonical: archivebox.machine.models.BinaryMachine.installed
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.installed
-```
-
-````
-
-````{py:attribute} tick
-:canonical: archivebox.machine.models.BinaryMachine.tick
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.tick
-```
-
-````
-
-````{py:method} can_install() -> bool
-:canonical: archivebox.machine.models.BinaryMachine.can_install
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.can_install
-```
-
-````
-
-````{py:method} enter_queued()
-:canonical: archivebox.machine.models.BinaryMachine.enter_queued
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.enter_queued
-```
-
-````
-
-````{py:method} on_install()
-:canonical: archivebox.machine.models.BinaryMachine.on_install
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.on_install
-```
-
-````
-
-````{py:method} enter_installed()
-:canonical: archivebox.machine.models.BinaryMachine.enter_installed
-
-```{autodoc2-docstring} archivebox.machine.models.BinaryMachine.enter_installed
-```
-
-````
-
-`````
-
-`````{py:class} ProcessMachine(obj, *args, **kwargs)
-:canonical: archivebox.machine.models.ProcessMachine
-
-Bases: {py:obj}`archivebox.workers.models.BaseStateMachine`
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine
-```
-
-```{rubric} Initialization
-```
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.__init__
-```
-
-````{py:attribute} model_attr_name
-:canonical: archivebox.machine.models.ProcessMachine.model_attr_name
-:value: >
-   'process'
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.model_attr_name
-```
-
-````
-
-````{py:attribute} process
-:canonical: archivebox.machine.models.ProcessMachine.process
-:type: archivebox.machine.models.Process
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.process
-```
-
-````
-
-````{py:attribute} queued
-:canonical: archivebox.machine.models.ProcessMachine.queued
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.queued
-```
-
-````
-
-````{py:attribute} running
-:canonical: archivebox.machine.models.ProcessMachine.running
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.running
-```
-
-````
-
-````{py:attribute} exited
-:canonical: archivebox.machine.models.ProcessMachine.exited
-:value: >
-   'State(...)'
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.exited
-```
-
-````
-
-````{py:attribute} tick
-:canonical: archivebox.machine.models.ProcessMachine.tick
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.tick
-```
-
-````
-
-````{py:attribute} launch
-:canonical: archivebox.machine.models.ProcessMachine.launch
-:value: >
-   'to(...)'
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.launch
-```
-
-````
-
-````{py:attribute} kill
-:canonical: archivebox.machine.models.ProcessMachine.kill
-:value: >
-   'to(...)'
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.kill
-```
-
-````
-
-````{py:method} can_start() -> bool
-:canonical: archivebox.machine.models.ProcessMachine.can_start
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.can_start
-```
-
-````
-
-````{py:method} is_exited() -> bool
-:canonical: archivebox.machine.models.ProcessMachine.is_exited
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.is_exited
-```
-
-````
-
-````{py:method} enter_queued()
-:canonical: archivebox.machine.models.ProcessMachine.enter_queued
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.enter_queued
-```
-
-````
-
-````{py:method} enter_running()
-:canonical: archivebox.machine.models.ProcessMachine.enter_running
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.enter_running
-```
-
-````
-
-````{py:method} enter_exited()
-:canonical: archivebox.machine.models.ProcessMachine.enter_exited
-
-```{autodoc2-docstring} archivebox.machine.models.ProcessMachine.enter_exited
-```
-
-````
-
-`````

docs/apidocs/archivebox/archivebox.services.runner.md

@@ -51,10 +51,6 @@
   - ```{autodoc2-docstring} archivebox.services.runner._is_nonfatal_setup_hook
     :summary:
     ```
-* - {py:obj}`_discover_archivebox_plugins <archivebox.services.runner._discover_archivebox_plugins>`
-  - ```{autodoc2-docstring} archivebox.services.runner._discover_archivebox_plugins
-    :summary:
-    ```
 * - {py:obj}`_runner_task_context <archivebox.services.runner._runner_task_context>`
   - ```{autodoc2-docstring} archivebox.services.runner._runner_task_context
     :summary:
@@ -63,10 +59,6 @@
   - ```{autodoc2-docstring} archivebox.services.runner._is_external_task_cancelled
     :summary:
     ```
-* - {py:obj}`_emit_machine_config <archivebox.services.runner._emit_machine_config>`
-  - ```{autodoc2-docstring} archivebox.services.runner._emit_machine_config
-    :summary:
-    ```
 * - {py:obj}`_run_event_now <archivebox.services.runner._run_event_now>`
   - ```{autodoc2-docstring} archivebox.services.runner._run_event_now
     :summary:
@@ -251,13 +243,6 @@
 ```
 ````
 
-````{py:function} _discover_archivebox_plugins() -> dict[str, abx_dl.models.Plugin]
-:canonical: archivebox.services.runner._discover_archivebox_plugins
-
-```{autodoc2-docstring} archivebox.services.runner._discover_archivebox_plugins
-```
-````
-
 ````{py:function} _runner_task_context() -> contextvars.Context
 :canonical: archivebox.services.runner._runner_task_context
 
@@ -272,14 +257,6 @@
 ```
 ````
 
-````{py:function} _emit_machine_config(bus, *, config: dict[str, typing.Any], derived_config: dict[str, typing.Any], parent_event=None) -> None
-:canonical: archivebox.services.runner._emit_machine_config
-:async:
-
-```{autodoc2-docstring} archivebox.services.runner._emit_machine_config
-```
-````
-
 ````{py:function} _run_event_now(event, timeout: float | None = None)
 :canonical: archivebox.services.runner._run_event_now
 :async:
@@ -342,14 +319,6 @@
 
 ````
 
-````{py:method} runtime_plugins() -> dict[str, abx_dl.models.Plugin]
-:canonical: archivebox.services.runner.CrawlRunner.runtime_plugins
-
-```{autodoc2-docstring} archivebox.services.runner.CrawlRunner.runtime_plugins
-```
-
-````
-
 ````{py:property} allow_maintenance_on_inactive_crawl
 :canonical: archivebox.services.runner.CrawlRunner.allow_maintenance_on_inactive_crawl
 :type: bool

docs/apidocs/archivebox/archivebox.workers.models.md

@@ -17,14 +17,8 @@
 
 * - {py:obj}`DefaultStatusChoices <archivebox.workers.models.DefaultStatusChoices>`
   -
-* - {py:obj}`ModelStateMachine <archivebox.workers.models.ModelStateMachine>`
-  -
-* - {py:obj}`BaseModelWithStateMachine <archivebox.workers.models.BaseModelWithStateMachine>`
-  -
-* - {py:obj}`ModelWithStateMachine <archivebox.workers.models.ModelWithStateMachine>`
-  -
-* - {py:obj}`BaseStateMachine <archivebox.workers.models.BaseStateMachine>`
-  - ```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine
+* - {py:obj}`ModelWithQueue <archivebox.workers.models.ModelWithQueue>`
+  - ```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue
     :summary:
     ```
 ````
@@ -67,19 +61,11 @@
   - ```{autodoc2-docstring} archivebox.workers.models.PACKAGE_ROOT
     :summary:
     ```
-* - {py:obj}`ObjectState <archivebox.workers.models.ObjectState>`
-  - ```{autodoc2-docstring} archivebox.workers.models.ObjectState
-    :summary:
-    ```
-* - {py:obj}`ObjectStateList <archivebox.workers.models.ObjectStateList>`
-  - ```{autodoc2-docstring} archivebox.workers.models.ObjectStateList
-    :summary:
-    ```
 ````
 
 ### API
 
-`````{py:class} DefaultStatusChoices(*args, **kwds)
+`````{py:class} DefaultStatusChoices()
 :canonical: archivebox.workers.models.DefaultStatusChoices
 
 Bases: {py:obj}`django.db.models.TextChoices`
@@ -208,550 +194,267 @@ Bases: {py:obj}`django.db.models.TextChoices`
 
 ````
 
-````{py:data} ObjectState
-:canonical: archivebox.workers.models.ObjectState
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.ObjectState
-```
-
-````
-
-````{py:data} ObjectStateList
-:canonical: archivebox.workers.models.ObjectStateList
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.ObjectStateList
-```
-
-````
-
-`````{py:class} ModelStateMachine
-:canonical: archivebox.workers.models.ModelStateMachine
-
-Bases: {py:obj}`typing.Protocol`
-
-````{py:method} tick() -> typing.Any
-:canonical: archivebox.workers.models.ModelStateMachine.tick
-
-```{autodoc2-docstring} archivebox.workers.models.ModelStateMachine.tick
-```
-
-````
-
-````{py:method} pause_requested() -> typing.Any
-:canonical: archivebox.workers.models.ModelStateMachine.pause_requested
-
-```{autodoc2-docstring} archivebox.workers.models.ModelStateMachine.pause_requested
-```
-
-````
-
-````{py:method} resume_requested() -> typing.Any
-:canonical: archivebox.workers.models.ModelStateMachine.resume_requested
-
-```{autodoc2-docstring} archivebox.workers.models.ModelStateMachine.resume_requested
-```
-
-````
-
-`````
-
-``````{py:class} BaseModelWithStateMachine(*args, **kwargs)
-:canonical: archivebox.workers.models.BaseModelWithStateMachine
+``````{py:class} ModelWithQueue(*args, **kwargs)
+:canonical: archivebox.workers.models.ModelWithQueue
 
 Bases: {py:obj}`django.db.models.Model`
 
-````{py:attribute} StatusChoices
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.StatusChoices
-:type: typing.ClassVar[type[archivebox.workers.models.DefaultStatusChoices]]
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.StatusChoices
-```
-
-````
-
-````{py:attribute} state_machine_name
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.state_machine_name
-:type: str | None
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.state_machine_name
-```
-
-````
-
-````{py:attribute} state_field_name
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.state_field_name
-:type: str
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.state_field_name
-```
-
-````
-
-````{py:attribute} state_machine_attr
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.state_machine_attr
-:type: str
-:value: >
-   'sm'
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.state_machine_attr
-```
-
-````
-
-````{py:attribute} bind_events_as_methods
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.bind_events_as_methods
-:type: bool
-:value: >
-   False
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.bind_events_as_methods
-```
-
-````
-
-````{py:attribute} warn_on_save_outside_runner
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.warn_on_save_outside_runner
-:type: typing.ClassVar[bool]
-:value: >
-   True
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.warn_on_save_outside_runner
-```
-
-````
-
-````{py:attribute} active_state
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.active_state
-:type: archivebox.workers.models.ObjectState
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.active_state
-```
-
-````
-
-````{py:attribute} retry_at_field_name
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.retry_at_field_name
-:type: str
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.retry_at_field_name
-```
-
-````
-
-`````{py:class} Meta
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta
-
-Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta`
-
-````{py:attribute} app_label
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta.app_label
-:value: >
-   'workers'
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.Meta.app_label
-```
-
-````
-
-````{py:attribute} abstract
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.Meta.abstract
-:value: >
-   True
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.Meta.abstract
-```
-
-````
-
-`````
-
-````{py:property} sm
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.sm
-:type: statemachine.StateMachine
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.sm
-```
-
-````
-
-````{py:method} status_counts(queryset: django.db.models.QuerySet | None = None, statuses: collections.abc.Iterable[str] | None = None) -> dict[str, int]
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.status_counts
-:classmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.status_counts
-```
-
-````
-
-````{py:method} check(sender=None, **kwargs)
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.check
-:classmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.check
-```
-
-````
-
-````{py:method} _state_to_str(state: archivebox.workers.models.ObjectState) -> str
-:canonical: archivebox.workers.models.BaseModelWithStateMachine._state_to_str
-:staticmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine._state_to_str
-```
-
-````
-
-````{py:property} RETRY_AT
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.RETRY_AT
-:type: datetime.datetime
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.RETRY_AT
-```
-
-````
-
-````{py:property} STATE
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.STATE
-:type: str
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.STATE
-```
-
-````
-
-````{py:method} bump_retry_at(seconds: int = 10)
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.bump_retry_at
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.bump_retry_at
-```
-
-````
-
-````{py:property} is_paused
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.is_paused
-:type: bool
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.is_paused
-```
-
-````
-
-````{py:method} safe_update(update_fields: dict[str, typing.Any], *, refresh: bool = True, extra_filter: dict[str, typing.Any] | None = None) -> bool
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.safe_update
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.safe_update
-```
-
-````
-
-````{py:method} save(*args, **kwargs)
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.save
-
-````
-
-````{py:method} pause(*, save: bool = True) -> bool
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.pause
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.pause
-```
-
-````
-
-````{py:method} resume(*, when: datetime.datetime | None = None, save: bool = True) -> bool
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.resume
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.resume
-```
-
-````
-
-````{py:method} update_and_requeue(*, refresh: bool = True, **kwargs) -> bool
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.update_and_requeue
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.update_and_requeue
-```
-
-````
-
-````{py:method} get_queue()
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.get_queue
-:classmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.get_queue
-```
-
-````
-
-````{py:method} claim_for_worker(obj: archivebox.workers.models.BaseModelWithStateMachine, lock_seconds: int = 60) -> bool
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.claim_for_worker
-:classmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.claim_for_worker
-```
-
-````
-
-````{py:method} claim_processing_lock(lock_seconds: int = 60) -> bool
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.claim_processing_lock
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.claim_processing_lock
-```
-
-````
-
-````{py:method} tick_claimed(lock_seconds: int = 60) -> bool
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.tick_claimed
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.tick_claimed
-```
-
-````
-
-````{py:method} ACTIVE_STATE() -> str
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.ACTIVE_STATE
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.ACTIVE_STATE
-```
-
-````
-
-````{py:method} INITIAL_STATE() -> str
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.INITIAL_STATE
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.INITIAL_STATE
-```
-
-````
-
-````{py:method} FINAL_STATES() -> list[str]
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.FINAL_STATES
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.FINAL_STATES
-```
-
-````
-
-````{py:method} FINAL_OR_ACTIVE_STATES() -> list[str]
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.FINAL_OR_ACTIVE_STATES
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.FINAL_OR_ACTIVE_STATES
-```
-
-````
-
-````{py:method} extend_choices(base_choices: type[django.db.models.TextChoices])
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.extend_choices
-:classmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.extend_choices
-```
-
-````
-
-````{py:method} StatusField(**kwargs) -> django.db.models.CharField
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.StatusField
-:classmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.StatusField
-```
-
-````
-
-````{py:method} RetryAtField(**kwargs) -> django.db.models.DateTimeField
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.RetryAtField
-:classmethod:
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.RetryAtField
-```
-
-````
-
-````{py:method} StateMachineClass() -> type[statemachine.StateMachine]
-:canonical: archivebox.workers.models.BaseModelWithStateMachine.StateMachineClass
-
-```{autodoc2-docstring} archivebox.workers.models.BaseModelWithStateMachine.StateMachineClass
-```
-
-````
-
-``````
-
-``````{py:class} ModelWithStateMachine(*args, **kwargs)
-:canonical: archivebox.workers.models.ModelWithStateMachine
-
-Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine`
-
-````{py:attribute} StatusChoices
-:canonical: archivebox.workers.models.ModelWithStateMachine.StatusChoices
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.StatusChoices
-```
-
-````
-
-````{py:attribute} status
-:canonical: archivebox.workers.models.ModelWithStateMachine.status
-:type: django.db.models.CharField
-:value: >
-   'StatusField(...)'
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.status
-```
-
-````
-
-````{py:attribute} retry_at
-:canonical: archivebox.workers.models.ModelWithStateMachine.retry_at
-:type: django.db.models.DateTimeField
-:value: >
-   'RetryAtField(...)'
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.retry_at
-```
-
-````
-
-````{py:attribute} state_machine_name
-:canonical: archivebox.workers.models.ModelWithStateMachine.state_machine_name
-:type: str | None
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.state_machine_name
-```
-
-````
-
-````{py:attribute} state_field_name
-:canonical: archivebox.workers.models.ModelWithStateMachine.state_field_name
-:type: str
-:value: >
-   'status'
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.state_field_name
-```
-
-````
-
-````{py:attribute} state_machine_attr
-:canonical: archivebox.workers.models.ModelWithStateMachine.state_machine_attr
-:type: str
-:value: >
-   'sm'
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.state_machine_attr
-```
-
-````
-
-````{py:attribute} bind_events_as_methods
-:canonical: archivebox.workers.models.ModelWithStateMachine.bind_events_as_methods
-:type: bool
-:value: >
-   False
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.bind_events_as_methods
-```
-
-````
-
-````{py:attribute} active_state
-:canonical: archivebox.workers.models.ModelWithStateMachine.active_state
-:value: >
-   None
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.active_state
-```
-
-````
-
-````{py:attribute} retry_at_field_name
-:canonical: archivebox.workers.models.ModelWithStateMachine.retry_at_field_name
-:type: str
-:value: >
-   'retry_at'
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.retry_at_field_name
-```
-
-````
-
-`````{py:class} Meta
-:canonical: archivebox.workers.models.ModelWithStateMachine.Meta
-
-Bases: {py:obj}`archivebox.workers.models.BaseModelWithStateMachine`
-
-````{py:attribute} abstract
-:canonical: archivebox.workers.models.ModelWithStateMachine.Meta.abstract
-:value: >
-   True
-
-```{autodoc2-docstring} archivebox.workers.models.ModelWithStateMachine.Meta.abstract
-```
-
-````
-
-`````
-
-``````
-
-`````{py:class} BaseStateMachine(obj, *args, **kwargs)
-:canonical: archivebox.workers.models.BaseStateMachine
-
-Bases: {py:obj}`statemachine.StateMachine`
-
-```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue
 ```
 
 ```{rubric} Initialization
 ```
 
-```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine.__init__
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.__init__
 ```
 
-````{py:attribute} model_attr_name
-:canonical: archivebox.workers.models.BaseStateMachine.model_attr_name
-:type: str
+````{py:attribute} StatusChoices
+:canonical: archivebox.workers.models.ModelWithQueue.StatusChoices
+:type: typing.ClassVar[type[django.db.models.TextChoices]]
 :value: >
-   'obj'
+   None
 
-```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine.model_attr_name
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.StatusChoices
 ```
 
 ````
 
-````{py:method} _register_callbacks(listeners: list[object])
-:canonical: archivebox.workers.models.BaseStateMachine._register_callbacks
+````{py:attribute} INITIAL_STATE
+:canonical: archivebox.workers.models.ModelWithQueue.INITIAL_STATE
+:type: typing.ClassVar[str]
+:value: >
+   None
 
-```{autodoc2-docstring} archivebox.workers.models.BaseStateMachine._register_callbacks
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.INITIAL_STATE
 ```
 
 ````
 
-````{py:method} __repr__() -> str
-:canonical: archivebox.workers.models.BaseStateMachine.__repr__
+````{py:attribute} ACTIVE_STATE
+:canonical: archivebox.workers.models.ModelWithQueue.ACTIVE_STATE
+:type: typing.ClassVar[str]
+:value: >
+   None
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.ACTIVE_STATE
+```
 
 ````
 
-````{py:method} __str__() -> str
-:canonical: archivebox.workers.models.BaseStateMachine.__str__
+````{py:attribute} FINAL_STATES
+:canonical: archivebox.workers.models.ModelWithQueue.FINAL_STATES
+:type: typing.ClassVar[tuple[str, ...]]
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.FINAL_STATES
+```
+
+````
+
+````{py:attribute} warn_on_save_outside_runner
+:canonical: archivebox.workers.models.ModelWithQueue.warn_on_save_outside_runner
+:type: typing.ClassVar[bool]
+:value: >
+   True
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.warn_on_save_outside_runner
+```
+
+````
+
+````{py:attribute} status
+:canonical: archivebox.workers.models.ModelWithQueue.status
+:type: django.db.models.CharField
+:value: >
+   'CharField(...)'
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.status
+```
+
+````
+
+````{py:attribute} retry_at
+:canonical: archivebox.workers.models.ModelWithQueue.retry_at
+:type: django.db.models.DateTimeField
+:value: >
+   'DateTimeField(...)'
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.retry_at
+```
+
+````
+
+`````{py:class} Meta
+:canonical: archivebox.workers.models.ModelWithQueue.Meta
+
+Bases: {py:obj}`django_stubs_ext.db.models.TypedModelMeta`
+
+````{py:attribute} app_label
+:canonical: archivebox.workers.models.ModelWithQueue.Meta.app_label
+:value: >
+   'workers'
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.Meta.app_label
+```
+
+````
+
+````{py:attribute} abstract
+:canonical: archivebox.workers.models.ModelWithQueue.Meta.abstract
+:value: >
+   True
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.Meta.abstract
+```
 
 ````
 
 `````
+
+````{py:attribute} FINAL_OR_ACTIVE_STATES
+:canonical: archivebox.workers.models.ModelWithQueue.FINAL_OR_ACTIVE_STATES
+:type: typing.ClassVar[tuple[str, ...]]
+:value: >
+   ()
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.FINAL_OR_ACTIVE_STATES
+```
+
+````
+
+````{py:method} status_counts(queryset: django.db.models.QuerySet | None = None, statuses: collections.abc.Iterable[str] | None = None) -> dict[str, int]
+:canonical: archivebox.workers.models.ModelWithQueue.status_counts
+:classmethod:
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.status_counts
+```
+
+````
+
+````{py:property} RETRY_AT
+:canonical: archivebox.workers.models.ModelWithQueue.RETRY_AT
+:type: datetime.datetime | None
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.RETRY_AT
+```
+
+````
+
+````{py:property} STATE
+:canonical: archivebox.workers.models.ModelWithQueue.STATE
+:type: str
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.STATE
+```
+
+````
+
+````{py:method} bump_retry_at(seconds: int = 10) -> None
+:canonical: archivebox.workers.models.ModelWithQueue.bump_retry_at
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.bump_retry_at
+```
+
+````
+
+````{py:property} is_paused
+:canonical: archivebox.workers.models.ModelWithQueue.is_paused
+:type: bool
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.is_paused
+```
+
+````
+
+````{py:method} safe_update(update_fields: dict[str, typing.Any], *, refresh: bool = True, extra_filter: dict[str, typing.Any] | None = None) -> bool
+:canonical: archivebox.workers.models.ModelWithQueue.safe_update
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.safe_update
+```
+
+````
+
+````{py:method} save(*args: typing.Any, **kwargs: typing.Any) -> None
+:canonical: archivebox.workers.models.ModelWithQueue.save
+
+````
+
+````{py:method} pause(*, save: bool = True) -> bool
+:canonical: archivebox.workers.models.ModelWithQueue.pause
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.pause
+```
+
+````
+
+````{py:method} resume(*, when: datetime.datetime | None = None, save: bool = True) -> bool
+:canonical: archivebox.workers.models.ModelWithQueue.resume
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.resume
+```
+
+````
+
+````{py:method} update_and_requeue(*, refresh: bool = True, **kwargs: typing.Any) -> bool
+:canonical: archivebox.workers.models.ModelWithQueue.update_and_requeue
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.update_and_requeue
+```
+
+````
+
+````{py:method} get_queue()
+:canonical: archivebox.workers.models.ModelWithQueue.get_queue
+:classmethod:
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.get_queue
+```
+
+````
+
+````{py:method} claim_for_worker(obj: archivebox.workers.models.ModelWithQueue, lock_seconds: int = 60) -> bool
+:canonical: archivebox.workers.models.ModelWithQueue.claim_for_worker
+:classmethod:
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.claim_for_worker
+```
+
+````
+
+````{py:method} claim_processing_lock(lock_seconds: int = 60) -> bool
+:canonical: archivebox.workers.models.ModelWithQueue.claim_processing_lock
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.claim_processing_lock
+```
+
+````
+
+````{py:method} extend_choices(base_choices: type[django.db.models.TextChoices])
+:canonical: archivebox.workers.models.ModelWithQueue.extend_choices
+:classmethod:
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.extend_choices
+```
+
+````
+
+````{py:method} StatusField(**kwargs: typing.Any) -> django.db.models.CharField
+:canonical: archivebox.workers.models.ModelWithQueue.StatusField
+:classmethod:
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.StatusField
+```
+
+````
+
+````{py:method} RetryAtField(**kwargs: typing.Any) -> django.db.models.DateTimeField
+:canonical: archivebox.workers.models.ModelWithQueue.RetryAtField
+:classmethod:
+
+```{autodoc2-docstring} archivebox.workers.models.ModelWithQueue.RetryAtField
+```
+
+````
+
+``````

docs/codeblocks.toml

@@ -5,12 +5,12 @@ version = 2
 [snippets]
 
 # README.md
-"7829a1f6ac113348-1" = "illustration"
+"1a488451c694b89d-1" = "illustration"
 "fe1f06a0a6b02c01-1" = "illustration"
 "dbbcf99c3829e104-1" = "run"
 "698f2b0f88796c9b-1" = "illustration"
-"398611ab1880ef51-1" = "illustration"
-"778c693286d7b895-1" = "illustration"
+"c39a1d40ec19b8d2-1" = "illustration"
+"ca67ec944b6385fa-1" = "illustration"
 "16f90085b2eb8756-1" = "illustration"
 "39bc0d99ed165be2-1" = "run"
 "d5ae3240405670bb-1" = "run"
@@ -23,8 +23,8 @@ version = 2
 "588be84a1f0fda9a-3" = "illustration"
 "1b9438e093866ff0-1" = "illustration"
 "6392c7b5f3593d07-1" = "run"
-"e244b54155aa0bff-1" = "run"
-"b5176e0b5ad1bcfc-1" = "illustration"
+"fe137daf1317bcd6-1" = "run"
+"724cf9fe3251b66f-1" = "illustration"
 "13fefe6d12cab753-1" = "illustration"
 "3b58e9b83d6c91fa-1" = "illustration"
 "4a595390c5fe50a4-1" = "illustration"
@@ -41,10 +41,10 @@ version = 2
 "fab7ffa7e26bc3e5-1" = "run"
 "06e963d9fad5b4f9-1" = "illustration"
 "834b852409be5697-1" = "illustration"
-"6cc5d78f24372d44-1" = "illustration"
+"b7ac41de5ec46791-1" = "illustration"
 "747d7bb0b608c5ae-1" = "illustration"
 "efbeb534feb1eaf8-1" = "illustration"
-"511f47bb050d2921-1" = "illustration"
+"498cf752a5b07259-1" = "illustration"
 "6f0ed916c341d2fc-1" = "illustration"
 "1e5a1e7eef7f2237-1" = "illustration"
 "3c4ecdabd2c9ba35-1" = "illustration"
@@ -74,8 +74,8 @@ version = 2
 # docs/ArchiveBox-Architecture-Diagrams.md
 "7442dd337fd0a1dd-1" = "illustration"
 "9d6f8ac3043ff691-1" = "illustration"
-"8c8802244e05cfb7-1" = "illustration"
-"d0eb428affe5bb44-1" = "illustration"
+"9529bb7bc396b6b0-1" = "illustration"
+"66018350af060c17-1" = "illustration"
 "7d1d068e1e754a75-1" = "illustration"
 
 # docs/Changelog.md
@@ -116,7 +116,7 @@ version = 2
 "150ae2c7f6d012ab-1" = "illustration"
 "8d86727108f1dbad-1" = "run"
 "6bcb6c2e8770907a-1" = "illustration"
-"6414a1bfaf0da999-1" = "illustration"
+"7616ed50e793ec24-1" = "illustration"
 "6353a7cf3014612e-1" = "illustration"
 "d66324df4c7f97c7-1" = "illustration"
 "17ed06e5b75aac1b-1" = "illustration"
@@ -232,9 +232,9 @@ version = 2
 "03803c44e54399c4-1" = "illustration"
 
 # docs/Upgrading.md
-"4c75ef58fc49ad99-1" = "illustration"
-"6678cb1b999c44ea-1" = "illustration"
-"23c840fa4bc05801-1" = "illustration"
+"ec110536c878a3be-1" = "illustration"
+"9282b1295b40bb26-1" = "illustration"
+"dcac3052373fc78c-1" = "illustration"
 "3075ca963b666b72-1" = "illustration"
 "c47a6f74d71e34c2-1" = "illustration"
 "24ce21d1e112e656-1" = "illustration"
@@ -263,7 +263,7 @@ version = 2
 "8fe4a2c15abb0ab2-1" = "system"
 "2b08779e4d24d305-1" = "system"
 "6392c7b5f3593d07-1" = "collection"
-"e244b54155aa0bff-1" = "docker"
+"fe137daf1317bcd6-1" = "docker"
 "88fb0039056c161e-1" = "collection"
 "d6cb543b1403f8ef-1" = "collection"
 "e4a4f50f45b47ec8-1" = "collection"
@@ -343,7 +343,7 @@ version = 2
 "8fe4a2c15abb0ab2-1" = "macos"
 "2b08779e4d24d305-1" = "macos"
 "6392c7b5f3593d07-1" = "ubuntu"
-"e244b54155aa0bff-1" = "docker"
+"fe137daf1317bcd6-1" = "docker"
 "88fb0039056c161e-1" = "ubuntu"
 "d6cb543b1403f8ef-1" = "ubuntu"
 "e4a4f50f45b47ec8-1" = "ubuntu"

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc366",
+  "version": "0.9.35rc405",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

old/TODO_hook_architecture.md

@@ -1379,7 +1379,7 @@ Background hooks are identified by `.bg.` suffix in filename:
 **Files to rename:**
 
 ```bash
-# Use .bg. suffix (not __background)
+# Use the .bg. suffix
 mv archivebox/plugins/consolelog/on_Snapshot__21_consolelog.js \
    archivebox/plugins/consolelog/on_Snapshot__21_consolelog.bg.js
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc366"
+version = "0.9.35rc405"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -49,8 +49,6 @@ dependencies = [
     "django-admin-data-views>=0.4.1",
     "django-object-actions>=4.3.0",
     "bleach>=6.2.0",          # for: stripping unsafe HTML from user-editable titles, notes, labels, tags
-    ### State Management
-    "python-statemachine[diagrams]>=2.3.6",
     ### CLI / Logging
     "click>=8.3.1",          # for: nicer CLI command + argument definitions
     "rich>=14.2.0",          # for: pretty CLI output
@@ -81,9 +79,9 @@ dependencies = [
     ### Extractor dependencies (runtime binaries resolved through abxpkg)
     ### Binary/Package Management
     "abxbus==2.5.56",                 # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abxpkg==1.12.110",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins==1.12.197",           # direct imports only; version constrained by abx-dl
-    "abx-dl==1.12.225",       # shared ArchiveBox downloader package
+    "abxpkg==1.12.117",                # direct imports only; version constrained by abx-dl -> abx-plugins
+    "abx-plugins==1.12.221",           # direct imports only; version constrained by abx-dl
+    "abx-dl==1.12.256",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
 ]
@@ -344,7 +342,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.35rc366"
+current_version = "v0.9.35rc405"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-08-27T06:11:53.268167371Z"
+exclude-newer = "2026-08-28T20:27:25.177666929Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-01T06:11:52.268183486Z", span = "PT1S" }
+abxbus = { timestamp = "2026-09-02T20:27:24.177684316Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.12.225"
+version = "1.12.256"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/5b/21/976fa0fdfdd6989b05cd328cd3d837e5638cdce5ea3644a13f07507fd13a/abx_dl-1.12.225.tar.gz", hash = "sha256:a886f51bfe9981cf6165ab6c65a4ec710756fc8a834e62b42ae3500ca578868c", size = 84223 }
+sdist = { url = "https://files.pythonhosted.org/packages/7a/80/c7c2555d3043b5b1c1e902fc1054941e8f866e5966146da84677637b54b1/abx_dl-1.12.256.tar.gz", hash = "sha256:f94da3ccbe6153aac46a9976d3de75fe5b6d2b576487c361e76fcbc2c967146b", size = 86752 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/af/94/8d91e0b97789ea4541ffc0241341903b36ec4a7b018e46a28baf48d9d737/abx_dl-1.12.225-py3-none-any.whl", hash = "sha256:62b6a6ae9e8087249ea1fc726fe5e747ad02340e59ca791a1d3590f613ec9e8c", size = 87900 },
+    { url = "https://files.pythonhosted.org/packages/8d/1e/0d0cd296587ec1390ddc166fec3bfbeca69845e9b06345cc1dd0ae13161a/abx_dl-1.12.256-py3-none-any.whl", hash = "sha256:091ac73f0ccf9eab18397897e1d46760df076106b3e2be16741e1145e2e66f49", size = 91257 },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.12.197"
+version = "1.12.221"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/11/ab/09ca5de37ea1505de4d7295ba2adbc4b83256a332a71079893153afa6f02/abx_plugins-1.12.197.tar.gz", hash = "sha256:d8ebbf8b47a9d5fb456717f414b12c84e09284a5e5fc6a91ffd4729b84b73c85", size = 259108, upload-time = "2026-09-01T06:03:06.121Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/99/06/5c806420d1267d9ba56cccd9f2ca2617e7aae0fe5fbef9678d08b7c9e0d6/abx_plugins-1.12.221.tar.gz", hash = "sha256:6d42a4eaf7cd8515977a23de9172ae05366243a4c4fb9b4d7d8d1acc6849fd3b", size = 258487, upload-time = "2026-09-02T20:19:21.765Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/30/a7/5b93c9bc8adb294df1f4415402bf5c7818d3f3a4a5c7196e3e3cdd043331/abx_plugins-1.12.197-py3-none-any.whl", hash = "sha256:e1533bddfc2f076e13c5a9c6b7124320a2863986d1a5e384dbc78da1e372f791", size = 412155, upload-time = "2026-09-01T06:03:08.224Z" },
+    { url = "https://files.pythonhosted.org/packages/05/fb/cf9c05a08bb006963b246ccce75dd3f6052954b7f110bb1f89ef02ff56d8/abx_plugins-1.12.221-py3-none-any.whl", hash = "sha256:43e2ce175c6ff4cc87d049b7b427ae22649e44bf2daca3de6e34ab6172acb337", size = 412364, upload-time = "2026-09-02T20:19:20.1Z" },
 ]
 
 [[package]]
@@ -75,7 +75,7 @@ wheels = [
 
 [[package]]
 name = "abxpkg"
-version = "1.12.110"
+version = "1.12.117"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -83,9 +83,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/dc/25/152a0e2dd37494cab5116629b4a3f26232dda6641e7b79f8eb7d1e9bfd75/abxpkg-1.12.110.tar.gz", hash = "sha256:94777e6ea76944b6577c46aaf4d845cdc68cfbdfe4c58ca24cd02071e367a660", size = 239950, upload-time = "2026-08-31T01:25:16.393Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/35/bbfaccae109328c716cea4e0cdbf2d2dc5975f448203bb3f4daa605591a6/abxpkg-1.12.117.tar.gz", hash = "sha256:4d88cc293f343eee2be7ecaab1203dfbc20571a4a39b2afa2f866c9ef8032dec", size = 239387, upload-time = "2026-09-02T19:57:26.142Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/05/95/49a113ceccf1d1ac33a90a0f8735abd43919f6cf2543917542acbdd61ea3/abxpkg-1.12.110-py3-none-any.whl", hash = "sha256:01e3e95c0af1a8fe775dee07eb16fa540a4e7007c3961a52e11d57bc2eae2430", size = 254023, upload-time = "2026-08-31T01:25:14.833Z" },
+    { url = "https://files.pythonhosted.org/packages/73/46/372be1f8fa915eb254ad865b97fb41073acdb5573dbce3e23a6083417d87/abxpkg-1.12.117-py3-none-any.whl", hash = "sha256:76ec8afaac7b2d7aece7bdda2bbf606e6373db51eb4dcb906008052d09e59577", size = 253749, upload-time = "2026-09-02T19:57:27.437Z" },
 ]
 
 [[package]]
@@ -120,7 +120,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.35rc366"
+version = "0.9.35rc405"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -149,7 +149,6 @@ dependencies = [
     { name = "py-machineid", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pydantic-settings", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-    { name = "python-statemachine", extra = ["diagrams"], marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -220,10 +219,10 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.12.225" },
-    { name = "abx-plugins", specifier = "==1.12.197" },
+    { name = "abx-dl", specifier = "==1.12.256" },
+    { name = "abx-plugins", specifier = "==1.12.221" },
     { name = "abxbus", specifier = "==2.5.56" },
-    { name = "abxpkg", specifier = "==1.12.110" },
+    { name = "abxpkg", specifier = "==1.12.117" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
     { name = "atomicwrites", specifier = "==1.4.1" },
     { name = "base32-crockford", specifier = ">=0.3.0" },
@@ -253,7 +252,6 @@ requires-dist = [
     { name = "pydantic", specifier = ">=2.8.0" },
     { name = "pydantic-settings", specifier = ">=2.5.2" },
     { name = "python-ldap", marker = "extra == 'ldap'", specifier = ">=3.4.3" },
-    { name = "python-statemachine", extras = ["diagrams"], specifier = ">=2.3.6" },
     { name = "requests", specifier = ">=2.32.3" },
     { name = "requests-tracker", marker = "extra == 'debug'", specifier = ">=0.3.3" },
     { name = "rich", specifier = ">=14.2.0" },
@@ -1798,18 +1796,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" },
 ]
 
-[[package]]
-name = "pydot"
-version = "4.0.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "pyparsing", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/50/35/b17cb89ff865484c6a20ef46bf9d95a5f07328292578de0b295f4a6beec2/pydot-4.0.1.tar.gz", hash = "sha256:c2148f681c4a33e08bf0e26a9e5f8e4099a82e0e2a068098f32ce86577364ad5", size = 162594, upload-time = "2025-06-17T20:09:56.454Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl", hash = "sha256:869c0efadd2708c0be1f916eb669f3d664ca684bc57ffb7ecc08e70d5e93fee6", size = 37087, upload-time = "2025-06-17T20:09:55.25Z" },
-]
-
 [[package]]
 name = "pygments"
 version = "2.20.0"
@@ -1831,15 +1817,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/54/18/1dd71c9b43192ab83f1d531ad6002dc81108ac36c475f79fb7a295abe2f4/pyopenssl-26.3.0-py3-none-any.whl", hash = "sha256:46367f8f66b92271e6d218da9c87607e1ef5a0bc5c8dea5bb3db82f395c385a3", size = 56008, upload-time = "2026-06-12T20:28:05.999Z" },
 ]
 
-[[package]]
-name = "pyparsing"
-version = "3.3.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" },
-]
-
 [[package]]
 name = "pyright"
 version = "1.1.411"
@@ -1974,20 +1951,6 @@ dependencies = [
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/b2/f4/60edeb794bbc9ed0ff2149bbaeec605f3ed331766459d195832ecbd0ba2d/python_ldap-3.4.7.tar.gz", hash = "sha256:bacd9fb680d20263d8570ade1cf234d90d281149a8beb4f079dd8f33f7613dc8", size = 387477, upload-time = "2026-05-20T13:41:04.358Z" }
 
-[[package]]
-name = "python-statemachine"
-version = "3.2.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a3/88/d24d2475069c96bbd752414863d8ffc26ef25f5179c87fda5869ba8cb5d5/python_statemachine-3.2.0.tar.gz", hash = "sha256:44b98cb9bb1081891ef6efa907c821d05cae5314a2b99713e56d56a9c962de08", size = 745821, upload-time = "2026-06-17T02:25:41.347Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d8/e3/25d1076e47554b559332bba62fb4abdc01704ea960f773c8db21b4ea1952/python_statemachine-3.2.0-py3-none-any.whl", hash = "sha256:8909916fc21208680f737d33c891a5331a1a0eeeaf3fe40efa278cb056855aee", size = 152809, upload-time = "2026-06-17T02:25:39.766Z" },
-]
-
-[package.optional-dependencies]
-diagrams = [
-    { name = "pydot", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-]
-
 [[package]]
 name = "pytz"
 version = "2026.2"