release: archivebox 0.9.34rc33

Dockerfile

@@ -318,6 +318,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends build-essential tesseract-ocr tesseract-ocr-eng \
     && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" gallery-dl \
+    && abxpkg install --no-cache --binproviders=pip --bin-dir="$LIB_DIR/env/bin" --overrides='{"pip":{"install_args":["--no-deps","forum-dl","chardet==5.2.0","pydantic==2.12.3","pydantic-core==2.41.4","typing-extensions>=4.14.1","annotated-types>=0.6.0","typing-inspection>=0.4.2","beautifulsoup4","soupsieve","lxml","requests","urllib3","certifi","idna","charset-normalizer","tenacity","python-dateutil","six","html2text","warcio"]}}' forum-dl \
     && if [ "$TARGETARCH" = "arm64" ]; then \
         abxpkg install --binproviders=npm --overrides='{"npm":{"install_args":["playwright@next"]}}' playwright; \
         abxpkg install --no-cache --install-timeout=600 --binproviders=playwright --bin-dir="$LIB_DIR/env/bin" chromium; \
@@ -337,6 +338,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$T
     && ln -sf "$(command -v node)" "$LIB_DIR/env/bin/node" \
     && ln -sf "$(command -v npm)" "$LIB_DIR/env/bin/npm" \
     && ln -sf "$(command -v java)" "$LIB_DIR/env/bin/java" \
+    && ln -sf "$(command -v git)" "$LIB_DIR/env/bin/git" \
     && ln -sf "$(command -v rg)" "$LIB_DIR/env/bin/rg" \
     && ln -sf "$(command -v sonic)" "$LIB_DIR/env/bin/sonic" \
     && find "$LIB_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
@@ -373,6 +375,7 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked,id=uv-$TARGETARCH$T
 # Initialize an empty image collection without rerunning dependency installs.
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
     && PUID=0 PGID=0 archivebox init \
     && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
     && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
@@ -394,8 +397,7 @@ RUN (echo -e "\n\n[√] Finished Docker build successfully. Saving build summary
 RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
     && chmod g+w "$TMP_DIR" "$LIB_DIR" "$LIB_DIR"/bin "$PLAYWRIGHT_BROWSERS_PATH" \
-    && ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install 2>&1 | tee -a /VERSION.txt \
-    && ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
+    && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury papersdl parse_rss_urls readability search_backend_sonic opendataloader search_backend_ripgrep 2>&1 | tee -a /VERSION.txt \
     && gosu "$ARCHIVEBOX_USER" archivebox version 2>&1 | tee -a /VERSION.txt \
     && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
     && find /venv "$CODE_DIR" "$LIB_DIR" "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \

Dockerfile.multistage

@@ -184,6 +184,7 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
 
 WORKDIR "$DATA_DIR"
 RUN echo "[+] Initializing image collection..." \
+    && find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + \
     && PUID=0 PGID=0 archivebox init \
     && find "$DATA_DIR" -type d -name __pycache__ -prune -exec rm -rf {} + \
     && find "$DATA_DIR" -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete \
@@ -198,7 +199,7 @@ RUN echo "[+] Initializing image collection..." \
 RUN chmod +x "$CODE_DIR"/bin/*.sh \
     && chown -R "$DEFAULT_PUID:$DEFAULT_PGID" "$LIB_DIR" \
     && chmod g+w "$TMP_DIR" "$LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
-    && ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
+    && GIT_BINARY="$LIB_DIR/env/bin/git" GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl" FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl" ABXPKG_INSTALL_TIMEOUT=600 ABXPKG_POSTINSTALL_SCRIPTS=True ABXPKG_MIN_RELEASE_AGE=0 TIMEOUT=600 gosu "$ARCHIVEBOX_USER" archivebox install archivewebpage defuddle forumdl gallerydl git istilldontcareaboutcookies liteparse mercury opendataloader papersdl parse_rss_urls readability search_backend_ripgrep search_backend_sonic 2>&1 | tee -a /VERSION.txt \
     && "$LIB_DIR/env/bin/chromium" --version | tee -a /VERSION.txt \
     && "$LIB_DIR/pip/packages/papers-dl/venv/bin/papers-dl" --version | tee -a /VERSION.txt \
     && /usr/bin/rg --version | head -1 | tee -a /VERSION.txt \

archivebox/__init__.py

@@ -79,16 +79,10 @@ def __getattr__(name: str):
         return VERSION
     if name in ("BUILTIN_PLUGINS_DIR", "USER_PLUGINS_DIR", "ALL_PLUGINS", "LOADED_PLUGINS"):
         from abx_plugins import get_plugins_dir
+        from .config.constants import CONSTANTS
 
         builtin_plugins_dir = Path(get_plugins_dir()).resolve()
-        user_plugins_dir = (
-            Path(
-                os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR")
-                or os.environ.get("USER_PLUGINS_DIR")
-                or os.environ.get("DATA_DIR", os.getcwd()),
-            )
-            / "custom_plugins"
-        )
+        user_plugins_dir = CONSTANTS.USER_PLUGINS_DIR
         plugins = {
             "builtin": builtin_plugins_dir,
             "user": user_plugins_dir,

archivebox/api/v1_api.py

@@ -68,14 +68,14 @@ class NinjaAPIWithIOCapture(NinjaAPI):
         # response['X-ArchiveBox-View'] = self.get_openapi_operation_id(request) or 'Unknown'
 
         # Add Auth Headers to response
-        api_token_attr = getattr(request, "_api_token", None)
+        api_token_attr = request.__dict__.get("_api_token")
         api_token = api_token_attr if isinstance(api_token_attr, APIToken) else None
         token_expiry = api_token.expires.isoformat() if api_token and api_token.expires else "Never"
 
-        response["X-ArchiveBox-Auth-Method"] = str(getattr(request, "_api_auth_method", "None"))
+        response["X-ArchiveBox-Auth-Method"] = str(request.__dict__.get("_api_auth_method", "None"))
         response["X-ArchiveBox-Auth-Expires"] = token_expiry
         response["X-ArchiveBox-Auth-Token-Id"] = str(api_token.id) if api_token else "None"
-        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if getattr(request.user, "pk", None) else "None"
+        response["X-ArchiveBox-Auth-User-Id"] = str(request.user.pk) if request.user.pk else "None"
         response["X-ArchiveBox-Auth-User-Username"] = request.user.username if isinstance(request.user, User) else "None"
 
         # import ipdb; ipdb.set_trace()

archivebox/api/v1_cli.py

@@ -150,8 +150,8 @@ def cli_add(request: HttpRequest, args: AddCommandSchema):
         "snapshot_ids": snapshot_ids,
         "queued_urls": args.urls,
     }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
 
     return {
         "success": True,
@@ -177,8 +177,8 @@ def cli_update(request: HttpRequest, args: UpdateCommandSchema):
         continuous=args.continuous,
         stop_daemon_stack=False,
     )
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -211,8 +211,8 @@ def cli_schedule(request: HttpRequest, args: ScheduleCommandSchema):
         config=config_overrides or None,
     )
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -249,8 +249,8 @@ def cli_search(request: HttpRequest, args: ListCommandSchema):
     elif args.as_csv:
         result_format = "csv"
 
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],
@@ -290,8 +290,8 @@ def cli_remove(request: HttpRequest, args: RemoveCommandSchema):
         "removed_snapshot_ids": removed_snapshot_ids,
         "remaining_snapshots": Snapshot.objects.count(),
     }
-    stdout = getattr(request, "stdout", None)
-    stderr = getattr(request, "stderr", None)
+    stdout = request.__dict__.get("stdout")
+    stderr = request.__dict__.get("stderr")
     return {
         "success": True,
         "errors": [],

archivebox/api/v1_core.py

@@ -273,7 +273,7 @@ def _parse_archiveresult_output_json(output_json: str | None) -> dict[str, Any]
 
 
 def _get_archiveresult_upload_data(request: HttpRequest):
-    cached = getattr(request, "_archiveresult_upload_data", None)
+    cached = request.__dict__.get("_archiveresult_upload_data")
     if cached is not None:
         return cached
 
@@ -468,12 +468,7 @@ def _write_archiveresult_files(
             )
 
         guessed_mime = mimetypes.guess_type(relative_output_path)[0]
-        output_mime_type = (
-            (mime_types[0] if mime_types else "")
-            or getattr(uploaded_file, "content_type", None)
-            or guessed_mime
-            or "application/octet-stream"
-        )
+        output_mime_type = (mime_types[0] if mime_types else "") or uploaded_file.content_type or guessed_mime or "application/octet-stream"
         output_files[relative_output_path] = {
             "extension": PurePosixPath(relative_output_path).suffix.lower().lstrip("."),
             "mimetype": output_mime_type,
@@ -501,7 +496,7 @@ def _write_archiveresult_files(
         guessed_mime = mimetypes.guess_type(saved_output_path)[0]
         output_mime_type = (
             (mime_types[index] if index < len(mime_types) else "")
-            or getattr(uploaded_file, "content_type", None)
+            or uploaded_file.content_type
             or guessed_mime
             or "application/octet-stream"
         )
@@ -702,7 +697,7 @@ class SnapshotSchema(Schema):
 
     @staticmethod
     def resolve_archiveresults(obj, context):
-        if bool(getattr(context["request"], "with_archiveresults", False)):
+        if bool(context["request"].__dict__.get("with_archiveresults", False)):
             return obj.archiveresult_set.all().distinct()
         return ArchiveResult.objects.none()
 
@@ -883,7 +878,7 @@ def get_snapshots(request: HttpRequest, filters: Query[SnapshotFilterSchema], wi
     if not query:
         return queryset
 
-    runtime_config = getattr(request, "archivebox_config", None)
+    runtime_config = request.archivebox_config
     search_mode = get_search_mode(filters.search_mode, config=runtime_config)
     try:
         return apply_snapshot_search(
@@ -1081,7 +1076,7 @@ class TagSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -1090,7 +1085,7 @@ class TagSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -1294,8 +1289,8 @@ def _public_tag_listing_enabled() -> bool:
 
 
 def _request_has_tag_autocomplete_access(request: HttpRequest) -> bool:
-    user = getattr(request, "user", None)
-    if getattr(user, "is_authenticated", False):
+    user = request.user
+    if user.is_authenticated:
         return True
 
     token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
@@ -1315,7 +1310,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     if not _request_has_tag_autocomplete_access(request):
         raise HttpError(401, "Authentication required")
 
-    public_only = not getattr(request.user, "is_authenticated", False) and not getattr(request, "_api_token", None)
+    public_only = not request.user.is_authenticated and not request.__dict__.get("_api_token")
     queryset = get_matching_tags(q)
     public_snapshots = public_snapshots_queryset(Snapshot.objects.all())
     if public_only:
@@ -1324,7 +1319,7 @@ def tags_autocomplete(request: HttpRequest, q: str = ""):
     add_snapshot_counts(tags, snapshot_queryset=public_snapshots if public_only else None)
 
     return {
-        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": getattr(tag, "num_snapshots", 0)} for tag in tags],
+        "tags": [{"id": tag.pk, "name": tag.name, "num_snapshots": tag.__dict__.get("num_snapshots", 0)} for tag in tags],
     }
 
 

archivebox/api/v1_crawls.py

@@ -58,7 +58,7 @@ class CrawlSchema(Schema):
     def resolve_created_by_username(obj):
         user_model = get_user_model()
         user = user_model.objects.get(id=obj.created_by_id)
-        username = getattr(user, "username", None)
+        username = user.username
         return username if isinstance(username, str) else str(user)
 
     @staticmethod
@@ -71,7 +71,7 @@ class CrawlSchema(Schema):
 
     @staticmethod
     def resolve_snapshots(obj, context):
-        if bool(getattr(context["request"], "with_snapshots", False)):
+        if bool(context["request"].__dict__.get("with_snapshots", False)):
             return obj.snapshot_set.all().distinct()
         return Snapshot.objects.none()
 
@@ -122,8 +122,7 @@ def create_crawl(request: HttpRequest, data: CrawlCreateSchema):
 
     tags = normalize_tag_list(data.tags, data.tags_str)
     config = dict(data.config or {})
-    request_user = request.user if request.user.is_authenticated else None
-    config.setdefault("PERMISSIONS", str(get_config(user=request_user).PERMISSIONS))
+    config.setdefault("PERMISSIONS", str(get_config().PERMISSIONS))
     crawl = Crawl.objects.create(
         urls="\n".join(urls),
         max_depth=data.max_depth,
@@ -164,8 +163,8 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     # Determine the effective viewer: session user takes precedence, otherwise
     # fall back to an API token passed via ?api_key=, X-ArchiveBox-API-Key, or
     # Authorization: Bearer ... (so that programmatic clients still work).
-    user = getattr(request, "user", None)
-    is_authenticated = bool(getattr(user, "is_authenticated", False) and getattr(user, "is_active", False))
+    user = request.user
+    is_authenticated = bool(user.is_authenticated and user.is_active)
     if not is_authenticated:
         token = request.GET.get("api_key") or request.headers.get("X-ArchiveBox-API-Key")
         auth_header = request.headers.get("Authorization", "")
@@ -183,7 +182,7 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
     # must be PUBLIC or UNLISTED. Don't disclose existence of private crawls.
     if not is_admin_user(request):
         permissions = normalize_permissions(crawl.permissions)
-        is_owner = bool(is_authenticated and getattr(crawl, "created_by_id", None) == getattr(user, "id", None))
+        is_owner = bool(is_authenticated and crawl.created_by_id == user.id)
         if not is_owner and permissions not in {PERMISSIONS_PUBLIC, PERMISSIONS_UNLISTED}:
             raise HttpError(404, "Crawl not found")
 

archivebox/api/v1_personas.py

@@ -136,7 +136,7 @@ def sync_persona(request: HttpRequest, payload: PersonaSyncSchema):
     created = persona is None
     if persona is None:
         persona = Persona(name=name)
-        if getattr(request.user, "is_authenticated", False):
+        if request.user.is_authenticated:
             persona.created_by = request.user
 
     persona.config = {

archivebox/base_models/admin.py

@@ -5,7 +5,7 @@ __package__ = "archivebox.base_models"
 import json
 import uuid
 from collections.abc import Mapping
-from typing import NotRequired, TypedDict
+from typing import NotRequired, TypedDict, cast
 
 from django import forms
 from django.contrib import admin
@@ -76,39 +76,23 @@ class KeyValueWidget(forms.Widget):
     def _get_config_options(self) -> dict[str, ConfigOption]:
         """Get available config options from plugins."""
         try:
-            from archivebox.config.common import ArchiveBoxConfig
-            from archivebox.plugins.discovery import discover_plugin_configs
+            from archivebox.config.common import config_field_metadata
 
             options: dict[str, ConfigOption] = {}
-            skipped_core_keys = {"ABX_RUNTIME", "DATA_DIR", "CRAWL_DIR", "SNAP_DIR"}
-            for key, field in ArchiveBoxConfig.model_fields.items():
-                if key in skipped_core_keys or key in ArchiveBoxConfig.computed_config_keys:
-                    continue
-                default = field.default
-                try:
-                    json.dumps(default)
-                except TypeError:
-                    default = str(default)
-                options[key] = {
-                    "plugin": "archivebox",
-                    "type": str(field.annotation),
-                    "default": default,
-                    "description": field.description or "",
+            for key, metadata in config_field_metadata().items():
+                option_type = metadata.get("type", "string")
+                option: ConfigOption = {
+                    "plugin": str(metadata.get("plugin", "archivebox")),
+                    "type": cast(str | list[str], option_type if isinstance(option_type, (str, list)) else str(option_type)),
+                    "default": metadata.get("default", ""),
+                    "description": str(metadata.get("description", "")),
                 }
-
-            plugin_configs = discover_plugin_configs()
-            for plugin_name, schema in plugin_configs.items():
-                for key, prop in schema.get("properties", {}).items():
-                    option: ConfigOption = {
-                        "plugin": plugin_name,
-                        "type": prop.get("type", "string"),
-                        "default": prop.get("default", ""),
-                        "description": prop.get("description", ""),
-                    }
+                schema = metadata.get("schema")
+                if isinstance(schema, Mapping):
                     for schema_key in ("enum", "pattern", "minimum", "maximum"):
-                        if schema_key in prop:
-                            option[schema_key] = prop[schema_key]
-                    options[key] = option
+                        if schema_key in schema:
+                            option[schema_key] = schema[schema_key]
+                options[key] = option
             return options
         except Exception:
             return {}
@@ -839,7 +823,7 @@ class ConfigEditorMixin(admin.ModelAdmin):
         """
         from archivebox.config.common import is_sensitive_config_key
 
-        if change and obj.pk and getattr(obj, "config", None) is not None:
+        if change and obj.pk and obj.config is not None:
             try:
                 stored = type(obj).objects.filter(pk=obj.pk).values_list("config", flat=True).first() or {}
             except Exception:

archivebox/base_models/models.py

@@ -20,6 +20,8 @@ from django.conf import settings
 
 from django_stubs_ext.db.models import TypedModelMeta
 
+from archivebox.config import CONSTANTS
+
 
 def normalize_config_json_values(config: Any) -> Any:
     if not isinstance(config, dict):
@@ -165,7 +167,7 @@ class ModelWithDeleteAfter(models.Model):
     def get_delete_after_config_value(self):
         from archivebox.config.common import get_config
 
-        return get_config(include_machine=False).DELETE_AFTER
+        return get_config(include_machine=False, resolve_plugins=False).DELETE_AFTER
 
     def set_delete_at_from_config(self, config_value=None) -> bool:
         if self.delete_at is not None:
@@ -253,9 +255,7 @@ class ModelWithOutputDir(ModelWithUUID):
 
     @classmethod
     def validate_output_paths_for_delete(cls, paths) -> tuple[Path, ...]:
-        from archivebox.config.common import get_config
-
-        data_dir = get_config().DATA_DIR.resolve()
+        data_dir = CONSTANTS.DATA_DIR.resolve()
         safe_paths = []
         for raw_path in paths:
             path = Path(raw_path)

archivebox/cli/__init__.py

@@ -139,9 +139,9 @@ class ArchiveBoxGroup(click.Group):
 
         # print(f'LAZY LOADING {import_path}')
         mod = import_module(modname)
-        func = getattr(mod, funcname)
+        func = vars(mod)[funcname]
 
-        if not hasattr(func, "__doc__"):
+        if func.__doc__ is None:
             raise ValueError(f"lazy loading of {import_path} failed - no docstring found on method")
 
         # if not isinstance(cmd, click.BaseCommand):

archivebox/cli/archivebox_add.py

@@ -113,11 +113,9 @@ def add(
     from archivebox.misc.logging_util import printable_filesize
     from archivebox.misc.system import get_dir_size
     from archivebox.core.shutdown_util import foreground_parent_watchdog, foreground_shutdown_signals
-    from django.contrib.auth import get_user_model
     from django.utils import timezone
 
     created_by_id = created_by_id or get_or_create_system_user_pk()
-    created_by = get_user_model().objects.filter(pk=created_by_id).first()
     started_at = timezone.now()
 
     if isinstance(urls, str):
@@ -151,7 +149,7 @@ def add(
     plugins = plugins or ""
     persona_obj, _ = Persona.objects.get_or_create(name=persona_name)
     persona_obj.ensure_dirs()
-    effective_persona_config = get_config(persona=persona_obj, user=created_by)
+    effective_persona_config = get_config(persona=persona_obj)
 
     crawl_config = {
         "PERMISSIONS": str(effective_persona_config.PERMISSIONS),
@@ -172,9 +170,8 @@ def add(
         **({"URL_DENYLIST": url_denylist} if url_denylist else {}),
     }
     # Caller-supplied overrides (e.g. {"ONLY_NEW": False}) are the highest
-    # priority — they win over persona/plugin/env defaults and get stamped
-    # directly onto crawl.config so the runtime resolution and admin UI both
-    # reflect them faithfully.
+    # priority for crawl-frozen keys. Runtime-derived execution keys are
+    # stripped by Crawl.save() and rederived when hooks run.
     crawl_config.update(config_overrides)
 
     crawl = Crawl.objects.create(
@@ -185,7 +182,7 @@ def add(
         label=f"{USER}@{HOSTNAME} $ {cmd_str} [{timestamp}]",
         created_by_id=created_by_id,
         status=Crawl.StatusChoices.QUEUED,
-        retry_at=None if (index_only or bg) else timezone.now(),
+        retry_at=None if index_only else timezone.now(),
         config=crawl_config,
     )
 

archivebox/cli/archivebox_config.py

@@ -58,7 +58,7 @@ def config(
             ]
             matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
             for config_section in CONFIGS.values():
-                aliases = getattr(config_section, "aliases", {})
+                aliases = {str(field.alias): field_name for field_name, field in type(config_section).model_fields.items() if field.alias}
 
                 for search_key in config_options:
                     # search all aliases in the section
@@ -90,7 +90,7 @@ def config(
 
         # Display core config sections
         for config_section in CONFIGS.values():
-            section_header = getattr(config_section, "toml_section_header", "")
+            section_header = config_section.toml_section_header
             if isinstance(section_header, str) and section_header:
                 print(f"[grey53]\\[{section_header}][/grey53]")
             else:

archivebox/cli/archivebox_extract.py

@@ -116,6 +116,7 @@ def run_plugins(
     wait: bool = True,
     emit_results: bool = True,
     show_progress: bool = True,
+    preserve_queued: bool = False,
 ) -> int:
     """
     Run plugins on Snapshots from input.
@@ -219,6 +220,18 @@ def run_plugins(
         else:
             requested_rows.add((snapshot_id, plugin_name, ""))
 
+    queued_rows: set[tuple[str, str, str]] = set()
+    if preserve_queued and requested_rows:
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+    rows_to_queue = requested_rows - queued_rows
+
     reset_fields = {
         "status": ArchiveResult.StatusChoices.QUEUED,
         "output_str": "",
@@ -230,23 +243,34 @@ def run_plugins(
         "end_ts": None,
         "modified_at": timezone.now(),
     }
-    if plugins_list:
-        ArchiveResult.objects.filter(snapshot_id__in=existing_snapshot_ids, plugin__in=plugins_list).update(**reset_fields)
-    elif requested_plugins_by_snapshot:
-        snapshot_ids_by_plugin: dict[str, set[str]] = defaultdict(set)
-        for snapshot_id, plugin_names in requested_plugins_by_snapshot.items():
-            if snapshot_id in existing_snapshot_ids:
-                for plugin_name in plugin_names:
-                    snapshot_ids_by_plugin[plugin_name].add(snapshot_id)
-        for plugin_name, plugin_snapshot_ids in snapshot_ids_by_plugin.items():
-            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name).update(**reset_fields)
-    existing_rows = set(
-        ArchiveResult.objects.filter(
-            snapshot_id__in=existing_snapshot_ids,
-            plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in requested_rows},
-        ).values_list("snapshot_id", "plugin", "hook_name"),
+    if rows_to_queue and plugins_list:
+        rows_to_reset_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            rows_to_reset_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in rows_to_reset_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    elif rows_to_queue and requested_plugins_by_snapshot:
+        snapshot_ids_by_hook: dict[tuple[str, str], set[str]] = defaultdict(set)
+        for snapshot_id, plugin_name, hook_name in rows_to_queue:
+            snapshot_ids_by_hook[(plugin_name, hook_name)].add(snapshot_id)
+        for (plugin_name, hook_name), plugin_snapshot_ids in snapshot_ids_by_hook.items():
+            ArchiveResult.objects.filter(snapshot_id__in=plugin_snapshot_ids, plugin=plugin_name, hook_name=hook_name).update(
+                **reset_fields,
+            )
+    existing_rows = (
+        {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=existing_snapshot_ids,
+                plugin__in={plugin_name for _snapshot_id, plugin_name, _hook_name in rows_to_queue},
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        if rows_to_queue
+        else set()
     )
-    missing_rows = requested_rows - {(str(snapshot_id), plugin_name, hook_name) for snapshot_id, plugin_name, hook_name in existing_rows}
+    missing_rows = rows_to_queue - existing_rows
     if missing_rows:
         ArchiveResult.objects.bulk_create(
             [
@@ -271,7 +295,23 @@ def run_plugins(
             # also keep status=paused here: `retry_at` only asks the orchestrator
             # to process the queued plugin rows, and run_due_snapshot restores
             # retry_at=MAX afterward instead of resuming the snapshot lifecycle.
-            for snapshot in Snapshot.objects.filter(id__in=existing_snapshot_ids).only("id", "status", "modified_at"):
+            affected_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in rows_to_queue}
+            if preserve_queued and queued_rows:
+                queued_snapshot_ids = {snapshot_id for snapshot_id, _plugin_name, _hook_name in queued_rows}
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids)
+                    .filter(retry_at__gt=queue_at)
+                    .values_list("id", flat=True)
+                )
+                affected_snapshot_ids.update(
+                    str(snapshot_id)
+                    for snapshot_id in Snapshot.objects.filter(id__in=queued_snapshot_ids, retry_at__isnull=True).values_list(
+                        "id",
+                        flat=True,
+                    )
+                )
+            for snapshot in Snapshot.objects.filter(id__in=affected_snapshot_ids).only("id", "status", "modified_at"):
                 # Guard the read-time status so we never bump retry_at on a
                 # row that's been re-queued / started by a concurrent runner.
                 snapshot.safe_update(

archivebox/cli/archivebox_help.py

@@ -85,8 +85,8 @@ def help() -> None:
     [link=https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration]https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration[/link]
 """)
 
-    config = get_config()
-    if os.access(config.ARCHIVE_DIR, os.R_OK) and config.ARCHIVE_DIR.is_dir():
+    get_config()
+    if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and CONSTANTS.ARCHIVE_DIR.is_dir():
         pretty_out_dir = str(CONSTANTS.DATA_DIR).replace(str(Path("~").expanduser()), "~")
         EXAMPLE_USAGE = f"""
 [light_slate_blue]DATA DIR[/light_slate_blue]: [yellow]{pretty_out_dir}[/yellow]

archivebox/cli/archivebox_init.py

@@ -25,7 +25,7 @@ def _display_data_path(path: Path, data_dir: Path) -> str:
 def init(force: bool = False, quick: bool = False, install: bool = False) -> None:
     """Initialize a new ArchiveBox collection in the current directory"""
 
-    from archivebox.config import CONSTANTS, VERSION, DATA_DIR
+    from archivebox.config import CONSTANTS, VERSION
     from archivebox.config.common import get_config
     from archivebox.config.collection import write_config_file
     from archivebox.misc.db import apply_migrations
@@ -37,7 +37,7 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     #     print("[red]:warning: This folder contains a JSON index. It is deprecated, and will no longer be kept up to date automatically.[/red]", file=sys.stderr)
     #     print("[red]    You can run `archivebox list --json --with-headers > static_index.json` to manually generate it.[/red]", file=sys.stderr)
 
-    is_empty = not len(set(os.listdir(DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
+    is_empty = not len(set(os.listdir(CONSTANTS.DATA_DIR)) - CONSTANTS.ALLOWED_IN_DATA_DIR)
     existing_index = os.path.isfile(CONSTANTS.DATABASE_FILE)
     if is_empty and not existing_index:
         print(f"[turquoise4][+] Initializing a new ArchiveBox v{VERSION} collection...[/turquoise4]")
@@ -65,23 +65,23 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     else:
         print("\n[green][+] Building archive folder structure...[/green]")
 
-    archive_path = _display_data_path(config.ARCHIVE_DIR, DATA_DIR)
-    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, DATA_DIR)
-    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, DATA_DIR)
+    archive_path = _display_data_path(CONSTANTS.ARCHIVE_DIR, CONSTANTS.DATA_DIR)
+    sources_path = _display_data_path(CONSTANTS.SOURCES_DIR, CONSTANTS.DATA_DIR)
+    logs_path = _display_data_path(CONSTANTS.LOGS_DIR, CONSTANTS.DATA_DIR)
     print(f"    + {archive_path}, {sources_path}, {logs_path}...")
     Path(CONSTANTS.SOURCES_DIR).mkdir(exist_ok=True)
-    config.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
-    config.USERS_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True)
+    CONSTANTS.USERS_DIR.mkdir(parents=True, exist_ok=True)
     Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)
-    for path in (Path(CONSTANTS.SOURCES_DIR), config.ARCHIVE_DIR, config.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
+    for path in (Path(CONSTANTS.SOURCES_DIR), CONSTANTS.ARCHIVE_DIR, CONSTANTS.USERS_DIR, Path(CONSTANTS.LOGS_DIR)):
         path.chmod(int(config.OUTPUT_PERMISSIONS, base=8) | 0o111)
 
-    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, DATA_DIR)}...")
+    print(f"    + {_display_data_path(CONSTANTS.CONFIG_FILE, CONSTANTS.DATA_DIR)}...")
 
     # create the .archivebox_id file with a unique ID for this collection
     from archivebox.config.paths import _get_collection_id
 
-    _get_collection_id(DATA_DIR, force_create=True)
+    _get_collection_id(CONSTANTS.DATA_DIR, force_create=True)
 
     # create the ArchiveBox.conf file
     write_config_file({"SECRET_KEY": config.SECRET_KEY})
@@ -96,12 +96,12 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
     setup_django()
     check_migrations(blocking=True, auto_apply=False)
 
-    for migration_line in apply_migrations(DATA_DIR):
+    for migration_line in apply_migrations(CONSTANTS.DATA_DIR):
         sys.stdout.write(f"    {migration_line}\n")
 
     assert os.path.isfile(CONSTANTS.DATABASE_FILE) and os.access(CONSTANTS.DATABASE_FILE, os.R_OK)
     print()
-    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, DATA_DIR)}")
+    print(f"    √ {_display_data_path(CONSTANTS.DATABASE_FILE, CONSTANTS.DATA_DIR)}")
 
     # from django.contrib.auth.models import User
     #     call_command("createsuperuser", interactive=True)

archivebox/cli/archivebox_install.py

@@ -22,12 +22,11 @@ def install(binaries: tuple[str, ...] = (), binproviders: str = "*", dry_run: bo
     """
 
     from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.misc.logging import stderr
     from archivebox.cli.archivebox_init import init
 
-    config = get_config()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
 
     if dry_run:
         print("[dim]Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow[/dim]")

archivebox/cli/archivebox_persona.py

@@ -372,7 +372,7 @@ def create_personas(
         if not is_tty:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -424,7 +424,7 @@ def list_personas(
         else:
             write_record(
                 {
-                    "id": str(persona.id) if hasattr(persona, "id") else None,
+                    "id": str(persona.id),
                     "name": persona.name,
                     "path": str(persona.path),
                     "CHROME_USER_DATA_DIR": persona.CHROME_USER_DATA_DIR,
@@ -500,7 +500,7 @@ def update_personas(name: str | None = None) -> int:
             if not is_tty:
                 write_record(
                     {
-                        "id": str(persona.id) if hasattr(persona, "id") else None,
+                        "id": str(persona.id),
                         "name": persona.name,
                         "path": str(persona.path),
                     },

archivebox/cli/archivebox_remove.py

@@ -12,7 +12,7 @@ import rich_click as click
 from django.db import OperationalError
 from django.db.models import QuerySet
 
-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.config.django import setup_django
 from archivebox.misc.util import enforce_types, docstring
 from archivebox.misc.checks import check_data_folder
@@ -33,7 +33,7 @@ def remove(
     after: float | None = None,
     before: float | None = None,
     yes: bool = False,
-    out_dir: Path = DATA_DIR,
+    out_dir: Path = CONSTANTS.DATA_DIR,
 ) -> QuerySet:
     """Remove the specified URLs from the archive"""
 

archivebox/cli/archivebox_run.py

@@ -40,14 +40,23 @@ Examples:
 __package__ = "archivebox.cli"
 __command__ = "archivebox run"
 
-import sys
 import asyncio
+import os
+import signal
+import sys
 from collections import defaultdict
 
 import rich_click as click
 from rich import print as rprint
 
 
+RUNNER_DAEMON_ENV = "ARCHIVEBOX_RUNNER_DAEMON"
+
+
+def _exit_daemon_runner_on_signal(sig: signal.Signals) -> None:
+    os._exit(128 + int(sig))
+
+
 def process_stdin_records() -> int:
     """
     Process JSONL records from stdin.
@@ -281,8 +290,16 @@ def run_runner(daemon: bool = False, crawl_id: str | None = None, maintenance_on
     # hook, continue/retry, second Ctrl+C exits" flow. Server/update/run owned
     # orchestrators should shut down immediately and cleanly on the first signal.
     interactive_interrupts = current.root.process_type == Process.TypeChoices.ADD
+    if daemon:
+        os.environ[RUNNER_DAEMON_ENV] = "1"
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=not daemon):
+        with (
+            foreground_shutdown_signals(
+                on_signal=_exit_daemon_runner_on_signal if daemon else None,
+                raise_on_first_signal=not daemon,
+            ),
+            foreground_parent_watchdog(enabled=not daemon),
+        ):
             run_pending_crawls(
                 daemon=daemon,
                 crawl_id=crawl_id,
@@ -322,7 +339,14 @@ def main(daemon: bool, crawl_id: str, snapshot_id: str, binary_id: str, maintena
 
     if daemon and not snapshot_id and not binary_id and not crawl_id:
         try:
-            with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=False):
+            os.environ[RUNNER_DAEMON_ENV] = "1"
+            with (
+                foreground_shutdown_signals(
+                    on_signal=_exit_daemon_runner_on_signal,
+                    raise_on_first_signal=False,
+                ),
+                foreground_parent_watchdog(enabled=False),
+            ):
                 sys.exit(run_runner(daemon=True, maintenance_only=maintenance_only))
         except KeyboardInterrupt:
             sys.exit(0)

archivebox/cli/archivebox_search.py

@@ -12,7 +12,7 @@ import rich_click as click
 
 from django.db.models import Q, QuerySet
 
-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.logging import stderr
 from archivebox.misc.util import enforce_types, docstring
@@ -146,7 +146,7 @@ def get_snapshots(
     filter_type: str = "substring",
     after: float | None = None,
     before: float | None = None,
-    out_dir: Path = DATA_DIR,
+    out_dir: Path = CONSTANTS.DATA_DIR,
 ) -> QuerySet["Snapshot", "Snapshot"]:
     """Filter and return Snapshots matching the given criteria."""
     from archivebox.core.models import Snapshot

archivebox/cli/archivebox_server.py

@@ -12,6 +12,7 @@ from collections.abc import Iterable
 import rich_click as click
 from rich import print
 
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import docstring, enforce_types
 
 
@@ -259,8 +260,6 @@ def server(
     host, port = _parse_and_validate_bind_spec(bind_spec)
 
     if daemonize and os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1":
-        from archivebox.config import CONSTANTS
-
         log_path = CONSTANTS.LOGS_DIR / "server.log"
         log_path.parent.mkdir(parents=True, exist_ok=True)
         daemon_env = os.environ.copy()
@@ -337,21 +336,25 @@ def server(
     runtime_config = get_config()
     _print_server_startup_warnings(runtime_config, host, port)
     bind_url = f"http://{host}:{port}"
-    command = current_command(Process.TypeChoices.SERVER, data_dir=config.DATA_DIR, url=bind_url)
+    command = current_command(Process.TypeChoices.SERVER, data_dir=CONSTANTS.DATA_DIR, url=bind_url)
 
     def still_owns_runtime_stack() -> bool:
         from django.db import connections
 
         try:
-            return command_owns_runtime_stack(command, data_dir=config.DATA_DIR)
+            return command_owns_runtime_stack(command, data_dir=CONSTANTS.DATA_DIR)
         finally:
             connections.close_all()
 
+    shutdown_state = None
     try:
-        with foreground_shutdown_signals(), foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"):
+        with (
+            foreground_shutdown_signals() as shutdown_state,
+            foreground_parent_watchdog(enabled=os.environ.get("ARCHIVEBOX_SERVER_DAEMON_CHILD") != "1"),
+        ):
             while True:
-                standby_result = standby_until_runtime_stack_needed(command, data_dir=config.DATA_DIR)
-                older_owner = runtime_stack_owner(data_dir=config.DATA_DIR, exclude_id=command.id)
+                standby_result = standby_until_runtime_stack_needed(command, data_dir=CONSTANTS.DATA_DIR)
+                older_owner = runtime_stack_owner(data_dir=CONSTANTS.DATA_DIR, exclude_id=command.id)
                 takeover_components = active_supervisord_runtime_components(config=config)
                 if older_owner and takeover_components:
                     print(
@@ -386,7 +389,8 @@ def server(
     except KeyboardInterrupt:
         pass
     finally:
-        command.mark_exited()
+        if not shutdown_state or not shutdown_state.signal_name:
+            command.mark_exited()
     print("\n[i][green][🟩] ArchiveBox server shut down gracefully.[/green][/i]")
 
 

archivebox/cli/archivebox_snapshot.py

@@ -313,13 +313,7 @@ def list_snapshots(
         for snapshot in queryset.iterator(chunk_size=500):
             if set(cols).issubset(simple_cols):
                 rows.append(
-                    ",".join(
-                        to_json(
-                            value.isoformat() if hasattr((value := getattr(snapshot, col, "")), "isoformat") else value,
-                            indent=None,
-                        )
-                        for col in cols
-                    ),
+                    ",".join(to_json(snapshot.serializable_value(col), indent=None) for col in cols),
                 )
             else:
                 rows.append(snapshot.to_csv(cols=cols, separator=","))

archivebox/cli/archivebox_status.py

@@ -8,7 +8,7 @@ import rich_click as click
 from rich import print
 
 from archivebox.misc.util import enforce_types, docstring
-from archivebox.config import DATA_DIR, CONSTANTS
+from archivebox.config import CONSTANTS
 from archivebox.config.common import get_config
 from archivebox.misc.system import get_dir_size
 from archivebox.misc.logging_util import printable_filesize
@@ -18,7 +18,7 @@ MAX_STATUS_FS_DIR_SCAN = 5000
 
 
 @enforce_types
-def status(out_dir: Path = DATA_DIR) -> None:
+def status(out_dir: Path = CONSTANTS.DATA_DIR) -> None:
     """Print out some info and statistics about the archive collection"""
 
     from django.contrib.auth import get_user_model
@@ -38,7 +38,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
 
     snapshots_qs = Snapshot.objects.all()
     num_sql_links = snapshots_qs.count()
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     legacy_snapshot_dirs = []
     if archive_dir.exists():
         legacy_snapshot_dirs = [
@@ -48,7 +48,7 @@ def status(out_dir: Path = DATA_DIR) -> None:
     print(f"    > JSON Link Details: {len(legacy_snapshot_dirs)} links".ljust(36), f"(found in {archive_dir.name}/*/index.json)")
     print()
     print("[green]\\[*] Scanning archive data directories...[/green]")
-    users_dir = config.USERS_DIR
+    users_dir = CONSTANTS.USERS_DIR
     scan_roots = [root for root in (archive_dir, users_dir) if root.exists()]
     scan_roots_display = ", ".join(str(root) for root in scan_roots) if scan_roots else str(archive_dir)
     print(f"[yellow]   {scan_roots_display}[/yellow]")

archivebox/cli/archivebox_update.py

@@ -105,9 +105,17 @@ def reindex_snapshots(
     wait_for_turn=None,
 ) -> dict[str, Any]:
     from archivebox.cli.archivebox_extract import run_plugins
+    from archivebox.core.models import ArchiveResult
+    from abx_dl.models import discover_plugins
 
-    stats: dict[str, Any] = {"processed": 0, "queued": 0, "reindexed": 0, "snapshot_ids": []}
+    stats: dict[str, Any] = {"processed": 0, "requested": 0, "queued": 0, "skipped_queued": 0, "reindexed": 0, "snapshot_ids": []}
     records: list[dict[str, str]] = []
+    plugins_by_name = discover_plugins()
+    required_hooks_by_plugin = {
+        plugin_name: frozenset(hook.name for hook in plugins_by_name[plugin_name].filter_hooks("Snapshot"))
+        for plugin_name in search_plugins
+        if plugin_name in plugins_by_name
+    }
 
     total = snapshots.count()
     print(f"[*] Reindexing {total} snapshots with search plugins: {', '.join(search_plugins)}")
@@ -118,6 +126,31 @@ def reindex_snapshots(
         if wait_for_turn:
             wait_for_turn()
         batch_records = list(records)
+        snapshot_ids = {record["snapshot_id"] for record in batch_records}
+        plugin_names = {record["plugin"] for record in batch_records}
+        queued_rows = {
+            (str(snapshot_id), plugin_name, hook_name)
+            for snapshot_id, plugin_name, hook_name in ArchiveResult.objects.filter(
+                snapshot_id__in=snapshot_ids,
+                plugin__in=plugin_names,
+                status=ArchiveResult.StatusChoices.QUEUED,
+            ).values_list("snapshot_id", "plugin", "hook_name")
+        }
+        records_to_queue = []
+        for record in batch_records:
+            snapshot_id = record["snapshot_id"]
+            plugin_name = record["plugin"]
+            required_hooks = required_hooks_by_plugin.get(plugin_name, frozenset())
+            if required_hooks and all((snapshot_id, plugin_name, hook_name) in queued_rows for hook_name in required_hooks):
+                stats["skipped_queued"] += 1
+                continue
+            records_to_queue.append(record)
+        if not records_to_queue:
+            print(
+                f"    [{stats['processed']}/{total}] Already queued {len(batch_records)} index jobs",
+            )
+            records.clear()
+            return
         # `archivebox update --index-only` intentionally breaks the usual
         # "runner discovers work" rule by inserting synthetic queued
         # ArchiveResult rows for search backends. run_plugins() keeps this as
@@ -127,15 +160,17 @@ def reindex_snapshots(
         # finish.
         exit_code = run_plugins(
             args=(),
-            records=batch_records,
+            records=records_to_queue,
             wait=False,
             emit_results=False,
             show_progress=False,
+            preserve_queued=True,
         )
         if exit_code != 0:
             raise SystemExit(exit_code)
+        stats["queued"] += len(records_to_queue)
         print(
-            f"    [{stats['processed']}/{total}] Queued {len(batch_records)} index jobs for orchestrator",
+            f"    [{stats['processed']}/{total}] Queued {len(records_to_queue)} index jobs for orchestrator",
         )
         records.clear()
 
@@ -156,7 +191,7 @@ def reindex_snapshots(
                         "plugin": plugin_name,
                     },
                 )
-                stats["queued"] += 1
+                stats["requested"] += 1
             if len(records) >= batch_size:
                 run_batch()
         except KeyboardInterrupt as err:
@@ -366,15 +401,55 @@ def update(
                             after=after,
                             resume=resume,
                         )
-                        stats = reindex_snapshots(
-                            snapshots,
-                            search_plugins=search_plugins,
-                            batch_size=batch_size,
-                            collect_ids=is_filtered_update,
-                            wait_for_turn=wait_for_turn,
+                        from django.db.models import Exists, OuterRef, Q
+                        from django.utils import timezone
+                        from archivebox.core.models import ArchiveResult, Snapshot
+
+                        scoped_snapshot_ids = snapshots.order_by().values("id") if is_filtered_update else None
+                        queued_index_results = ArchiveResult.objects.filter(
+                            status=ArchiveResult.StatusChoices.QUEUED,
+                            plugin__in=search_plugins,
                         )
-                        print_index_stats(stats)
-                        touched_snapshot_ids.update(stats.get("snapshot_ids", []))
+                        if scoped_snapshot_ids is not None:
+                            queued_index_results = queued_index_results.filter(snapshot_id__in=scoped_snapshot_ids)
+
+                        if queued_index_results.exists():
+                            now = timezone.now()
+                            queued_result_for_snapshot = queued_index_results.filter(snapshot_id=OuterRef("pk"))
+                            snapshots_to_wake = (
+                                Snapshot.objects.filter(
+                                    status__in=(Snapshot.StatusChoices.SEALED, Snapshot.StatusChoices.PAUSED),
+                                )
+                                .annotate(
+                                    has_queued_index_result=Exists(queued_result_for_snapshot),
+                                )
+                                .filter(
+                                    has_queued_index_result=True,
+                                )
+                                .filter(
+                                    Q(retry_at__isnull=True) | Q(retry_at__gt=now),
+                                )
+                            )
+                            if scoped_snapshot_ids is not None:
+                                snapshots_to_wake = snapshots_to_wake.filter(id__in=scoped_snapshot_ids)
+                            woken_count = snapshots_to_wake.update(
+                                retry_at=now,
+                                modified_at=now,
+                            )
+                            print(
+                                "[*] Existing queued search index jobs found; "
+                                f"skipping backfill scan and waking {woken_count} snapshot(s) for the runner.",
+                            )
+                        else:
+                            stats = reindex_snapshots(
+                                snapshots,
+                                search_plugins=search_plugins,
+                                batch_size=batch_size,
+                                collect_ids=is_filtered_update,
+                                wait_for_turn=wait_for_turn,
+                            )
+                            print_index_stats(stats)
+                            touched_snapshot_ids.update(stats.get("snapshot_ids", []))
 
                 if do_run_until_idle and (do_index or not ran_post_migrate_runner):
                     # Search/index backfill intentionally queues targeted
@@ -401,7 +476,7 @@ def update(
                 resume = None
     except (KeyboardInterrupt, asyncio.CancelledError) as err:
         exit_code = 130
-        exact_resume = getattr(err, "archivebox_resume", None)
+        exact_resume = err.__dict__.get("archivebox_resume")
         resume_cmd = ["archivebox", "update"]
         if migrate_only:
             resume_cmd.append("--migrate-only")
@@ -464,7 +539,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     1:1 mapping between DB and filesystem.
     """
     from archivebox.core.models import Snapshot
-    from archivebox.config.common import get_config
+    from archivebox.config import CONSTANTS
     from archivebox.crawls.models import Crawl
     from django.utils import timezone
 
@@ -473,8 +548,7 @@ def drain_old_archive_dirs(resume_from: str | None = None, batch_size: int = 500
     crawl_url_sets: dict[str, set[str]] = {}
     dirty_crawl_ids: set[str] = set()
 
-    runtime_config = get_config()
-    archive_dir = runtime_config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     if not archive_dir.exists():
         return stats
 
@@ -905,7 +979,9 @@ def print_index_stats(stats: dict[str, Any]) -> None:
     print(f"""
 [green]Search Reindex Complete[/green]
   Scanned rows:      {stats["processed"]}
+  Requested jobs:    {stats.get("requested", stats["queued"])}
   Queued index jobs: {stats["queued"]}
+  Already queued:    {stats.get("skipped_queued", 0)}
 """)
 
 

archivebox/cli/archivebox_version.py

@@ -32,7 +32,6 @@ def _format_binary_abspath(
         (lib_dir, "LIB_DIR/"),
         (Path(os.environ.get("LIB_DIR", "")), "LIB_DIR/") if os.environ.get("LIB_DIR") else (Path(), ""),
         (personas_dir, "PERSONAS_DIR/"),
-        (Path(os.environ.get("PERSONAS_DIR", "")), "PERSONAS_DIR/") if os.environ.get("PERSONAS_DIR") else (Path(), ""),
         (home, "~/"),
     )
 
@@ -155,7 +154,7 @@ def version(
     )
     prnt()
 
-    if not (os.access(config.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
+    if not (os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) and os.access(CONSTANTS.CONFIG_FILE, os.R_OK)):
         PANEL_TEXT = "\n".join(
             (
                 "",
@@ -219,7 +218,7 @@ def version(
                             installed.abspath,
                             pwd=Path.cwd(),
                             lib_dir=config.LIB_DIR,
-                            personas_dir=config.PERSONAS_DIR,
+                            personas_dir=CONSTANTS.PERSONAS_DIR,
                             home=Path.home(),
                         )
                         if compact_paths
@@ -275,7 +274,7 @@ def version(
             prnt(f"  [red]Error getting code locations: {e}[/red]")
 
         prnt()
-        if os.access(config.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
+        if os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK) or os.access(CONSTANTS.CONFIG_FILE, os.R_OK):
             prnt("[bright_yellow][i] Data locations:[/bright_yellow]")
             try:
                 for name, path in get_data_locations().items():

archivebox/config/collection.py

@@ -3,6 +3,7 @@ __package__ = "archivebox.config"
 import io
 import json
 import os
+from collections.abc import Mapping
 from typing import Any
 
 from archivebox.config.constants import CONSTANTS
@@ -54,7 +55,7 @@ def _coerce_to_str_dict(config: Any) -> dict[str, str]:
     """
     if not config:
         return {}
-    if not hasattr(config, "items"):
+    if not isinstance(config, Mapping):
         return {}
     flat: dict[str, str] = {}
     for key, value in config.items():

archivebox/config/common.py

@@ -7,12 +7,15 @@ import os
 import re
 import secrets
 import sys
+import time
 import shutil
-from functools import lru_cache
+import inspect
+from functools import lru_cache, wraps
 from collections.abc import Mapping
 from datetime import timedelta
 from typing import Any, ClassVar, cast
 from pathlib import Path
+from urllib.parse import quote
 
 from rich.console import Console
 from pydantic import BaseModel, Field, PrivateAttr, create_model, field_validator, model_validator
@@ -30,6 +33,7 @@ from .permissions import IN_DOCKER
 ConfigOverrides = Mapping[str, object]
 ConfigPayload = dict[str, object]
 PluginSchemaDocuments = dict[str, dict[str, Any]]
+LIVE_CONFIG_BASE_URL = "/admin/environment/config/"
 
 ###################### Config ##########################
 
@@ -38,6 +42,24 @@ _STDERR_CONSOLE = Console(stderr=True)
 _WARNED_ARCHIVING_CONFIGS: set[tuple[int, bool]] = set()
 
 
+def _perf_trace(label):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                return func(*args, **kwargs)
+            started_at = time.perf_counter()
+            try:
+                return func(*args, **kwargs)
+            finally:
+                elapsed_ms = (time.perf_counter() - started_at) * 1000
+                print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+        return wrapper
+
+    return decorator
+
+
 def _legacy_bool(value: object) -> bool | None:
     if value is None:
         return None
@@ -64,6 +86,16 @@ def permissions_from_legacy_public_flags(raw_config: Mapping[str, object]) -> st
     return None
 
 
+def resolve_delete_after_config_value(*configs: Mapping[str, Any] | None) -> str:
+    for config in configs:
+        if config is None:
+            continue
+        value = config.get("DELETE_AFTER")
+        if value:
+            return str(value)
+    return "0"
+
+
 _SENSITIVE_CONFIG_KEY_NEEDLES = ("TOKEN", "SECRET", "API_KEY", "APIKEY", "PASSWORD")
 SENSITIVE_CONFIG_VALUE_REDACTED = "********"
 _SCOPE_CRAWL_FROZEN = "crawl_frozen"
@@ -118,8 +150,15 @@ def redact_sensitive_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
     return redacted
 
 
-def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | None) -> dict[str, Any]:
-    """Return a JSON-safe config dict suitable for storage or event payloads."""
+def normalize_runtime_config(
+    config: BaseConfigSet | Mapping[str, Any] | str | None,
+    *,
+    only_crawl_execution: bool = False,
+    exclude_runtime_derived: bool = False,
+    exclude_crawl_execution: bool = False,
+    json_safe: bool = True,
+) -> dict[str, Any]:
+    """Return config filtered for runtime/frozen usage, optionally JSON-safe."""
     if config is None:
         return {}
     if isinstance(config, BaseConfigSet):
@@ -128,21 +167,51 @@ def normalize_runtime_config(config: BaseConfigSet | Mapping[str, Any] | str | N
         config = json.loads(config)
     else:
         config = dict(config)
-    return {key: value for key, value in json.loads(json.dumps(config, default=str)).items() if value is not None}
+
+    runtime_derived_keys = ArchiveBoxConfig.runtime_derived_config_keys() if exclude_runtime_derived else frozenset()
+    filtered = {
+        key: value
+        for key, value in config.items()
+        if (
+            value is not None
+            and (not only_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) == _SCOPE_CRAWL_EXECUTION)
+            and (not exclude_runtime_derived or str(key) not in runtime_derived_keys)
+            and (not exclude_crawl_execution or ArchiveBoxConfig.scope_for_key(str(key)) != _SCOPE_CRAWL_EXECUTION)
+        )
+    }
+    if not json_safe:
+        return filtered
+    return {key: value for key, value in json.loads(json.dumps(filtered, default=str)).items() if value is not None}
 
 
 def build_crawl_config_snapshot(
     *,
-    user: Any = None,
     persona: Any = None,
     overrides: Mapping[str, Any] | None = None,
     base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
 ) -> dict[str, Any]:
-    """Build the frozen runtime config stored on Crawl.config at creation time."""
-    effective = get_config(user=user, persona=persona, base_config=base_config)
-    frozen = effective.for_crawl_frozen()
+    """Build the frozen crawl config stored on Crawl.config at creation time."""
+    explicit_overrides = set(overrides or {})
+    plugin_owned_keys = set(_plugin_config_properties(PLUGIN_CONFIG_SCHEMAS)) - set(ArchiveBoxBaseConfig.model_fields)
+    effective = get_config(persona=persona, base_config=base_config)
+    frozen = effective.for_crawl_frozen(persona=persona)
+    if persona is not None:
+        persona_config = persona.get_derived_config()
+        for key in plugin_owned_keys - explicit_overrides:
+            if key in persona_config:
+                frozen.pop(key, None)
     if overrides:
-        frozen = get_config(base_config=frozen, overrides=overrides, include_machine=False).for_crawl_frozen()
+        resolved = get_config(base_config=frozen, overrides=overrides, include_machine=False)
+        resolved_payload = normalize_runtime_config(resolved)
+        frozen = resolved.for_crawl_frozen(persona=persona)
+        for key in plugin_owned_keys & explicit_overrides:
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_FROZEN and key in resolved_payload:
+                frozen[key] = resolved_payload[key]
+        if persona is not None:
+            persona_config = persona.get_derived_config()
+            for key in plugin_owned_keys - explicit_overrides:
+                if key in persona_config:
+                    frozen.pop(key, None)
     return frozen
 
 
@@ -187,11 +256,6 @@ class StorageConfig(BaseConfigSet):
     toml_section_header: str = "STORAGE_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    # ARCHIVE_DIR / USERS_DIR are resolved dynamically via get_config().
-    ARCHIVE_DIR: Path = Field(default=CONSTANTS.ARCHIVE_DIR)
-    USERS_DIR: Path = Field(default=CONSTANTS.USERS_DIR)
-    PERSONAS_DIR: Path = Field(default=CONSTANTS.PERSONAS_DIR)
-
     # TMP_DIR must be a local, fast, readable/writable dir by archivebox user,
     # must be a short path due to unix path length restrictions for socket files (<100 chars)
     # must be a local SSD/tmpfs for speed and because bind mounts/network mounts/FUSE dont support unix sockets
@@ -206,10 +270,6 @@ class StorageConfig(BaseConfigSet):
     # Runtime lookup must use provider-specific paths under LIB_DIR instead.
     LIB_BIN_DIR: Path = Field(default=CONSTANTS.DEFAULT_LIB_BIN_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
-    # CUSTOM_TEMPLATES_DIR allows users to override default templates
-    # defaults to DATA_DIR / 'user_templates' but can be configured
-    CUSTOM_TEMPLATES_DIR: Path = Field(default=CONSTANTS.CUSTOM_TEMPLATES_DIR)
-
     OUTPUT_PERMISSIONS: str = Field(default="644")
     ENFORCE_ATOMIC_WRITES: bool = Field(default=True)
     ALLOW_NO_UNIX_SOCKETS: bool = Field(default=False, alias="ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS")
@@ -244,8 +304,6 @@ class ServerConfig(BaseConfigSet):
     FOOTER_INFO: str = Field(
         default="Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.",
     )
-    # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
-
     PUBLIC_INDEX: bool = Field(default=True)
     PUBLIC_ADD_VIEW: bool = Field(default=False)
 
@@ -355,7 +413,7 @@ class ArchivingConfig(BaseConfigSet):
     URL_DENYLIST: str = Field(default=r"\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$", alias="URL_BLACKLIST")
     URL_ALLOWLIST: str | None = Field(default=None, alias="URL_WHITELIST")
 
-    DEFAULT_PERSONA: str = Field(default="Default")
+    DEFAULT_PERSONA: str = Field(default="Default", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     PERMISSIONS: str = Field(
         default="public",
         description="Snapshot visibility: public lists and serves content, unlisted serves direct links only, private requires admin login.",
@@ -449,7 +507,7 @@ class SearchBackendConfig(BaseConfigSet):
     toml_section_header: str = "SEARCH_BACKEND_CONFIG"
     _scope: str = PrivateAttr(default=_SCOPE_SERVER)
 
-    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep")
+    SEARCH_BACKEND_ENGINE: str = Field(default="ripgrep", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
 
 
 def _plugin_user_config_value(value: Any) -> str:
@@ -469,9 +527,7 @@ def _discover_plugin_config_schemas() -> PluginSchemaDocuments:
 
     schemas: PluginSchemaDocuments = {}
     if BASE_CONFIG_PATH.exists():
-        schemas["base"] = {
-            "properties": json.loads(BASE_CONFIG_PATH.read_text()).get("properties", {}),
-        }
+        schemas["base"] = json.loads(BASE_CONFIG_PATH.read_text())
     schemas.update(discover_plugin_configs())
     return schemas
 
@@ -524,10 +580,6 @@ class ArchiveBoxBaseConfig(
         populate_by_name=True,
     )
 
-    DATA_DIR: Path = Field(default=CONSTANTS.DATA_DIR, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    ABX_RUNTIME: str = Field(default="archivebox", json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    CRAWL_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
-    SNAP_DIR: Path | None = Field(default=None, json_schema_extra={"scope": _SCOPE_CRAWL_EXECUTION})
     computed_config_keys: ClassVar[tuple[str, ...]] = COMPUTED_CONFIG_KEYS
 
     @classmethod
@@ -566,68 +618,161 @@ class ArchiveBoxBaseConfig(
 
     @classmethod
     def _plugin_field_scope(cls, key: str) -> str | None:
+        scope = None
         for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
             properties = schema.get("properties") if isinstance(schema, dict) else None
             if not isinstance(properties, dict) or key not in properties:
                 continue
             prop_schema = properties.get(key) or {}
             if isinstance(prop_schema, Mapping) and prop_schema.get("x-scope"):
-                return str(prop_schema["x-scope"])
-            if str(plugin_name).startswith("search_backend_"):
-                return _SCOPE_SERVER
-            return _SCOPE_CRAWL_FROZEN
-        return None
+                scope = str(prop_schema["x-scope"])
+            elif scope is None:
+                scope = _SCOPE_CRAWL_FROZEN
+        return scope
 
     @classmethod
+    @lru_cache(maxsize=None)
     def scope_for_key(cls, key: str) -> str:
+        for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+            properties = schema.get("properties") if isinstance(schema, dict) else None
+            if isinstance(properties, dict) and key == f"{str(plugin_name).upper()}_ENABLED" and key in properties:
+                return _SCOPE_CRAWL_EXECUTION
+        if key.endswith("_BINARY"):
+            return _SCOPE_CRAWL_EXECUTION
         return cls._core_field_scope(key) or cls._plugin_field_scope(key) or _SCOPE_SERVER
 
-    def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
-        allowed_scopes = {_SCOPE_CRAWL_FROZEN}
-        if include_execution:
-            allowed_scopes.add(_SCOPE_CRAWL_EXECUTION)
-        return {key: value for key, value in normalize_runtime_config(self).items() if type(self).scope_for_key(key) in allowed_scopes}
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _scope_by_key(cls) -> dict[str, str]:
+        return {key: cls.scope_for_key(key) for key in cls.model_fields}
 
-    def for_crawl_execution(self) -> dict[str, Any]:
-        """Config safe to pass to crawl/snapshot hook execution."""
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_frozen_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_FROZEN)
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def _crawl_runtime_keys(cls) -> frozenset[str]:
+        return frozenset(key for key, scope in cls._scope_by_key().items() if scope in {_SCOPE_CRAWL_FROZEN, _SCOPE_CRAWL_EXECUTION})
+
+    @classmethod
+    @lru_cache(maxsize=1)
+    def runtime_derived_config_keys(cls) -> frozenset[str]:
+        runtime_derived_keys = {
+            "ABX_INSTALL_CACHE",
+            "ACTIVE_PERSONA",
+            "CHROME_DOWNLOADS_DIR",
+            "CHROME_USER_DATA_DIR",
+            "DEFAULT_PERSONA",
+            "EXTRA_CONTEXT",
+        }
+        return frozenset(
+            key for key, scope in cls._scope_by_key().items() if scope == _SCOPE_CRAWL_EXECUTION and key in runtime_derived_keys
+        )
+
+    def _scoped_config(self, *, include_execution: bool) -> dict[str, Any]:
+        keys = type(self)._crawl_runtime_keys() if include_execution else type(self)._crawl_frozen_keys()
+        payload = self.model_dump(mode="json")
+        return {key: payload[key] for key in keys if payload.get(key) is not None}
+
+    def for_crawl(self) -> dict[str, Any]:
+        """Config scoped to crawl execution, without runtime object overlays."""
         return self._scoped_config(include_execution=True)
 
-    def for_crawl_frozen(self) -> dict[str, Any]:
+    def for_crawl_frozen(self, *, persona: Any = None) -> dict[str, Any]:
         """Config safe to persist permanently on Crawl.config."""
-        return self._scoped_config(include_execution=False)
+        frozen = self._scoped_config(include_execution=False)
+        if persona is not None:
+            persona_config = dict(persona.config or {})
+            scope_by_key = type(self)._scope_by_key()
+            for key in persona.get_derived_config():
+                if key not in persona_config and scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    frozen.pop(key, None)
+        return frozen
+
+    @_perf_trace("archivebox.config.for_crawl_runtime")
+    def for_crawl_runtime(
+        self,
+        *,
+        crawl: Any = None,
+        snapshot: Any = None,
+        persona: Any = None,
+        runtime_overrides: Mapping[str, Any] | None = None,
+        extra_context: Mapping[str, Any] | None = None,
+        crawl_output_dir: Any = None,
+        snapshot_output_dir: Any = None,
+    ) -> dict[str, Any]:
+        """Config payload safe to pass to crawl/snapshot hook execution."""
+        config = self.for_crawl()
+        scope_by_key = type(self)._scope_by_key()
+        model_fields = type(self).model_fields
+        for key in type(self).runtime_derived_config_keys():
+            config.pop(key, None)
+        # ArchiveBox owns SEARCH_BACKEND_ENGINE and uses it during model
+        # validation to derive the selected backend's *_ENABLED flag. Hooks
+        # only receive the backend-local flags, never the selector itself.
+        config.pop("SEARCH_BACKEND_ENGINE", None)
+        if persona is not None:
+            for key, value in persona.get_derived_config().items():
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+
+        if crawl is not None:
+            for key, value in dict(crawl.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["CRAWL_DIR"] = str(crawl_output_dir if crawl_output_dir is not None else crawl.output_dir)
+
+        if snapshot is not None:
+            for key, value in dict(snapshot.config or {}).items():
+                if key in model_fields and scope_by_key.get(key) != _SCOPE_CRAWL_EXECUTION:
+                    config[key] = value
+            config["SNAP_DIR"] = str(snapshot_output_dir if snapshot_output_dir is not None else snapshot.output_dir)
+
+        if runtime_overrides:
+            config.update(normalize_runtime_config(runtime_overrides, json_safe=False))
+
+        if extra_context:
+            context: dict[str, Any] = {}
+            if config.get("EXTRA_CONTEXT"):
+                parsed_extra_context = json.loads(str(config["EXTRA_CONTEXT"]))
+                if not isinstance(parsed_extra_context, dict):
+                    raise TypeError("EXTRA_CONTEXT must decode to an object")
+                context = parsed_extra_context
+            context.update(dict(extra_context))
+            config["EXTRA_CONTEXT"] = json.dumps(context, separators=(",", ":"), sort_keys=True)
+
+        return config
 
     @model_validator(mode="after")
     def resolve_runtime_paths(self):
-        self.DATA_DIR = self.DATA_DIR.expanduser().resolve()
-
-        archive_dir = self.ARCHIVE_DIR.expanduser()
-        if archive_dir == (CONSTANTS.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME) and self.DATA_DIR != CONSTANTS.DATA_DIR:
-            archive_dir = self.DATA_DIR / CONSTANTS.ARCHIVE_DIR_NAME
-        if not archive_dir.is_absolute():
-            archive_dir = self.DATA_DIR / archive_dir
-        self.ARCHIVE_DIR = archive_dir.resolve()
-
-        users_dir = self.USERS_DIR.expanduser()
-        if users_dir == (CONSTANTS.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME):
-            users_dir = self.ARCHIVE_DIR / CONSTANTS.USERS_DIR_NAME
-        if not users_dir.is_absolute():
-            users_dir = self.ARCHIVE_DIR / users_dir
-        self.USERS_DIR = users_dir.resolve()
-
         lib_dir = self.LIB_DIR.expanduser()
         if not lib_dir.is_absolute():
-            lib_dir = self.DATA_DIR / lib_dir
+            lib_dir = CONSTANTS.DATA_DIR / lib_dir
         self.LIB_DIR = lib_dir.resolve()
 
         lib_bin_dir = self.LIB_BIN_DIR.expanduser()
         if lib_bin_dir == CONSTANTS.DEFAULT_LIB_BIN_DIR and self.LIB_DIR != CONSTANTS.DEFAULT_LIB_DIR:
             lib_bin_dir = self.LIB_DIR / "bin"
         elif not lib_bin_dir.is_absolute():
-            lib_bin_dir = self.DATA_DIR / lib_bin_dir
+            lib_bin_dir = CONSTANTS.DATA_DIR / lib_bin_dir
         self.LIB_BIN_DIR = lib_bin_dir.resolve()
 
         return self
 
+    @model_validator(mode="after")
+    def derive_plugin_enabled_config(self):
+        plugin_names = _normalize_plugins_config_value(self.PLUGINS)
+        selected_plugins = _plugins_with_required_plugins(plugin_names) if plugin_names else set()
+        search_backend = self.SEARCH_BACKEND_ENGINE.strip().lower()
+        if search_backend:
+            selected_plugins.add(f"search_backend_{search_backend}")
+        for plugin_name, enabled_key in _plugin_enabled_config_keys().items():
+            if plugin_names or plugin_name in selected_plugins:
+                setattr(self, enabled_key, plugin_name in selected_plugins)
+        return self
+
 
 def _build_archivebox_config_model(plugin_schemas: PluginSchemaDocuments) -> type[ArchiveBoxBaseConfig]:
     core_fields = set(ArchiveBoxBaseConfig.model_fields)
@@ -649,15 +794,196 @@ PLUGIN_CONFIG_SCHEMAS = _discover_plugin_config_schemas()
 ArchiveBoxConfig = _build_archivebox_config_model(PLUGIN_CONFIG_SCHEMAS)
 
 
+def _normalize_plugins_config_value(value: Any) -> set[str]:
+    if value is None:
+        return set()
+    if isinstance(value, str):
+        raw = value.strip()
+        if not raw:
+            return set()
+        if raw.startswith("["):
+            try:
+                parsed = json.loads(raw)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, list):
+                return {str(plugin).strip().lower() for plugin in parsed if str(plugin).strip()}
+        return {plugin.strip().lower() for plugin in raw.split(",") if plugin.strip()}
+    if isinstance(value, (list, tuple, set)):
+        return {str(plugin).strip().lower() for plugin in value if str(plugin).strip()}
+    normalized = str(value).strip().lower()
+    return {normalized} if normalized else set()
+
+
+@lru_cache(maxsize=1)
+def _plugin_enabled_config_keys() -> dict[str, str]:
+    enabled_keys: dict[str, str] = {}
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        enabled_key = f"{str(plugin_name).upper()}_ENABLED"
+        if enabled_key in properties and ArchiveBoxConfig.scope_for_key(enabled_key) == _SCOPE_CRAWL_EXECUTION:
+            enabled_keys[str(plugin_name).lower()] = enabled_key
+    return enabled_keys
+
+
+def _plugins_with_required_plugins(plugin_names: set[str]) -> set[str]:
+    selected = set(plugin_names)
+    pending = list(selected)
+    while pending:
+        plugin_name = pending.pop()
+        schema = PLUGIN_CONFIG_SCHEMAS.get(plugin_name, {})
+        required_plugins = schema.get("required_plugins") if isinstance(schema, dict) else None
+        if not isinstance(required_plugins, list):
+            continue
+        for required_plugin in required_plugins:
+            required_plugin_name = str(required_plugin).strip().lower()
+            if required_plugin_name and required_plugin_name not in selected:
+                selected.add(required_plugin_name)
+                pending.append(required_plugin_name)
+    return selected
+
+
+def get_live_config_url(key: str) -> str:
+    return f"{LIVE_CONFIG_BASE_URL}{quote(key)}/"
+
+
+@lru_cache(maxsize=1)
+def config_field_metadata() -> dict[str, dict[str, Any]]:
+    """Return one centralized metadata map for core and plugin config fields."""
+    metadata: dict[str, dict[str, Any]] = {}
+    for key, field in ArchiveBoxConfig.model_fields.items():
+        if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION or key in ArchiveBoxConfig.computed_config_keys:
+            continue
+        default = field.default
+        try:
+            json.dumps(default)
+        except TypeError:
+            default = str(default)
+        metadata[key] = {
+            "plugin": "archivebox",
+            "section": find_config_section(key),
+            "type": config_field_type(key),
+            "default": default,
+            "description": field.description or "",
+            "scope": ArchiveBoxConfig.scope_for_key(key),
+            "sensitive": is_sensitive_config_key(key),
+        }
+    for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items():
+        properties = schema.get("properties") if isinstance(schema, dict) else None
+        if not isinstance(properties, dict):
+            continue
+        for key, prop in properties.items():
+            if not isinstance(prop, Mapping):
+                continue
+            if ArchiveBoxConfig.scope_for_key(key) == _SCOPE_CRAWL_EXECUTION:
+                continue
+            metadata[key] = {
+                **metadata.get(key, {}),
+                "plugin": plugin_name,
+                "section": "PLUGINS",
+                "type": prop.get("type", metadata.get(key, {}).get("type", "string")),
+                "default": prop.get("default", metadata.get(key, {}).get("default", "")),
+                "description": prop.get("description", metadata.get(key, {}).get("description", "")),
+                "scope": ArchiveBoxConfig.scope_for_key(key),
+                "sensitive": bool(prop.get("x-sensitive")) or is_sensitive_config_key(key),
+                "schema": dict(prop),
+            }
+    return metadata
+
+
+def find_config_section(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return "CONSTANT"
+    for section_id, section in get_all_configs().items():
+        if key in type(section).model_fields:
+            return section_id
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "PLUGINS"
+    return "DYNAMIC"
+
+
+def find_config_default(key: str) -> str:
+    from archivebox.config import CONSTANTS_CONFIG
+
+    if key in CONSTANTS_CONFIG:
+        return str(CONSTANTS_CONFIG[key])
+
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return ""
+    default_val = field.default
+    if callable(default_val):
+        default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
+        if default_val.count(")") > default_val.count("("):
+            default_val = default_val[:-1]
+    else:
+        default_val = str(default_val)
+    return default_val
+
+
+def config_field_type(key: str) -> str:
+    field = ArchiveBoxConfig.model_fields.get(key)
+    if field is None:
+        return "str"
+    annotation = field.annotation
+    try:
+        return annotation.__name__
+    except AttributeError:
+        return str(annotation)
+
+
+def find_config_type(key: str) -> str:
+    return config_field_type(key)
+
+
+def find_config_source(key: str, merged_config: Mapping[str, Any]) -> str:
+    """Determine where a config value comes from."""
+    try:
+        from archivebox.machine.models import Machine
+
+        machine = Machine.current()
+        if machine.config and key in machine.config:
+            return "Machine"
+    except Exception:
+        pass
+
+    if key in os.environ:
+        return "Environment"
+
+    file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
+    if key in file_config:
+        return "File"
+
+    if key in _plugin_config_properties(PLUGIN_CONFIG_SCHEMAS):
+        return "Plugin Default"
+
+    return "Default"
+
+
+def get_request_config(request: Any, *, resolve_plugins: bool = False) -> ArchiveBoxBaseConfig:
+    """Return the per-request ArchiveBox config, upgrading to plugin resolution if needed."""
+    request_state = request.__dict__
+    request_config = request_state.get("archivebox_config")
+    request_config_resolves_plugins = bool(request_state.get("_archivebox_config_resolves_plugins", False))
+    if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
+        request_config = get_config(resolve_plugins=resolve_plugins)
+        request.archivebox_config = request_config
+        request._archivebox_config_resolves_plugins = resolve_plugins
+    return request_config
+
+
+@_perf_trace("archivebox.config.get_config")
 def get_config(
     defaults: ConfigOverrides | None = None,
     overrides: ConfigOverrides | None = None,
     base_config: ArchiveBoxBaseConfig | Mapping[str, object] | None = None,
     persona: Any = None,
-    user: Any = None,
     crawl: Any = None,
     snapshot: Any = None,
-    archiveresult: Any = None,
     machine: Any = None,
     include_machine: bool = True,
     resolve_plugins: bool = True,
@@ -666,28 +992,20 @@ def get_config(
     """
     Get merged config from all sources.
 
-    Priority (highest to lowest):
-    1. Explicit overrides
-    2. Per-ArchiveResult config
-    3. Per-snapshot config and output path
-    4. Frozen per-crawl config and output path
-    5. Per-user config (only when resolving outside a crawl)
-    6. Per-persona derived config (only when resolving outside a crawl)
-    7. Current machine derived config (only when resolving outside a crawl)
-    8. Environment variables (only when resolving outside a crawl)
-    9. Config file (ArchiveBox.conf, only when resolving outside a crawl)
-    10. Plugin schema defaults
-    11. Core config defaults
-    """
-    if snapshot is None and archiveresult is not None:
-        snapshot = archiveresult.snapshot
+    Defaults are hydrated by pydantic from core/plugin defaults,
+    ArchiveBox.conf, and environment variables. Persisted Machine/Persona
+    values then apply for live crawl-execution scope, while Crawl/Snapshot
+    rows apply their frozen crawl-scope values. Explicit overrides win last.
 
+    Crawl-execution config is not persisted on Crawl.config. It is rederived
+    from current Machine/Persona state and hydrated process defaults each time.
+    """
     if crawl is None and snapshot is not None:
         crawl = snapshot.crawl
 
     crawl_config_base = crawl is not None and base_config is None
 
-    if include_machine and machine is None and not crawl_config_base:
+    if include_machine and machine is None:
         try:
             from django.apps import apps
 
@@ -698,58 +1016,68 @@ def get_config(
         except Exception:
             machine = None
 
-    if persona is None and crawl is not None and not crawl_config_base:
+    if persona is None and crawl is not None:
         persona = crawl.resolve_persona()
 
     config_data: ConfigPayload = dict(defaults or {})
     base_config_payload: ConfigPayload = {}
     if crawl_config_base:
-        config_data.update(dict(crawl.config or {}))
+        config_data.update(
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
+        config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
     elif base_config is not None:
         if isinstance(base_config, ArchiveBoxBaseConfig):
             base_config_payload.update(base_config.model_dump(mode="json"))
         else:
             base_config_payload.update(dict(base_config))
-        config_data.update(base_config_payload)
+        config_data.update(normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False))
     else:
-        config_data.update(ArchiveBoxConfig().model_dump(mode="json"))
+        config_data.update(
+            normalize_runtime_config(ArchiveBoxConfig().model_dump(mode="json"), exclude_runtime_derived=True, json_safe=False),
+        )
         legacy_permissions = permissions_from_legacy_public_flags({**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **os.environ})
         if legacy_permissions:
             config_data["PERMISSIONS"] = legacy_permissions
 
     scope_overrides: ConfigPayload = {}
 
-    if not crawl_config_base:
-        if include_machine and machine is not None and machine.config:
-            from archivebox.machine.models import _sanitize_machine_config
+    if include_machine and machine is not None and machine.config:
+        from archivebox.machine.models import _sanitize_machine_config
 
-            scope_overrides.update(_sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")))
+        scope_overrides.update(
+            normalize_runtime_config(
+                _sanitize_machine_config(machine.config, lib_dir=config_data.get("LIB_DIR")),
+                only_crawl_execution=crawl_config_base,
+                exclude_runtime_derived=True,
+                json_safe=False,
+            ),
+        )
 
-        if persona is not None:
-            scope_overrides.update(persona.get_derived_config())
-
-        user_config = getattr(user, "config", None)
-        if user_config:
-            scope_overrides.update(user_config)
+    if persona is not None:
+        persona_config = normalize_runtime_config(
+            persona.get_derived_config(),
+            exclude_runtime_derived=not crawl_config_base,
+            json_safe=False,
+        )
+        if crawl_config_base:
+            scope_by_key = ArchiveBoxConfig._scope_by_key()
+            crawl_keys = set(dict(crawl.config or {}))
+            persona_config = {
+                key: value
+                for key, value in persona_config.items()
+                if scope_by_key.get(key) == _SCOPE_CRAWL_EXECUTION or key not in crawl_keys
+            }
+        scope_overrides.update(persona_config)
 
     if crawl is not None and crawl.config and not crawl_config_base:
-        scope_overrides.update(crawl.config)
-
-    if crawl is not None:
-        if not overrides or "CRAWL_DIR" not in overrides:
-            scope_overrides["CRAWL_DIR"] = crawl.output_dir
+        scope_overrides.update(normalize_runtime_config(crawl.config, exclude_crawl_execution=True, json_safe=False))
 
     if snapshot is not None and snapshot.config:
-        scope_overrides.update(snapshot.config)
-
-    if snapshot is not None:
-        scope_overrides["SNAP_DIR"] = snapshot.output_dir
-
-    if archiveresult is not None and archiveresult.config:
-        scope_overrides.update(archiveresult.config)
+        scope_overrides.update(normalize_runtime_config(snapshot.config, exclude_crawl_execution=True, json_safe=False))
 
     if overrides:
-        scope_overrides.update(overrides)
+        scope_overrides.update(normalize_runtime_config(overrides, exclude_crawl_execution=True, json_safe=False))
 
     legacy_scope_permissions = permissions_from_legacy_public_flags(scope_overrides)
     if legacy_scope_permissions:
@@ -759,28 +1087,46 @@ def get_config(
     config_data.update(archivebox_scope_overrides)
 
     if resolve_plugins:
-        plugin_schemas = {
-            plugin_name: schema.get("properties", {}) for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)
-        }
+        plugin_schemas = {plugin_name: schema for plugin_name, schema in PLUGIN_CONFIG_SCHEMAS.items() if isinstance(schema, dict)}
         plugin_global_config = {key: str(value) if isinstance(value, Path) else value for key, value in config_data.items()}
-        plugin_user_config = _plugin_user_config(scope_overrides)
+        plugin_user_config = _plugin_user_config(
+            {
+                **normalize_runtime_config(config_data, only_crawl_execution=True, json_safe=False),
+                **scope_overrides,
+            },
+        )
         if not crawl_config_base:
-            plugin_user_config = {**BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE), **plugin_user_config}
+            plugin_user_config = {
+                **normalize_runtime_config(
+                    BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE),
+                    exclude_runtime_derived=True,
+                    json_safe=False,
+                ),
+                **plugin_user_config,
+            }
         plugin_sections = resolve_plugin_configs(
             plugin_schemas,
             global_config=plugin_global_config,
             user_config=plugin_user_config,
+            environ={},
         )
         for plugin_config in plugin_sections.values():
-            config_data.update(plugin_config)
+            for key, value in plugin_config.items():
+                if key in ArchiveBoxBaseConfig.model_fields and key not in archivebox_scope_overrides and key not in base_config_payload:
+                    continue
+                config_data[key] = value
         if base_config_payload:
-            config_data.update({key: value for key, value in base_config_payload.items() if key in _archivebox_config_input_names()})
+            config_data.update(
+                {
+                    key: value
+                    for key, value in normalize_runtime_config(base_config_payload, exclude_runtime_derived=True, json_safe=False).items()
+                    if key in _archivebox_config_input_names()
+                },
+            )
         if crawl_config_base:
-            config_data.update(dict(crawl.config or {}))
+            config_data.update(normalize_runtime_config(dict(crawl.config or {}), exclude_crawl_execution=True, json_safe=False))
         config_data.update(archivebox_scope_overrides)
 
-    config_data["ABX_RUNTIME"] = "archivebox"
-
     # Decode JSON-encoded complex values (dict/list fields) that came from
     # string-only sources before validation. ``IniConfigSettingsSource`` does
     # this for the ArchiveBox.conf path, but Machine.config (mirrored from the
@@ -807,7 +1153,7 @@ def get_config(
     config = ArchiveBoxConfig.model_validate(config_data)
     if redact_sensitive:
         for key in type(config).model_fields:
-            value = getattr(config, key, None)
+            value = config[key]
             if is_sensitive_config_key(key) and value not in (None, ""):
                 setattr(config, key, SENSITIVE_CONFIG_VALUE_REDACTED)
     os.environ["LIB_DIR"] = str(config.LIB_DIR)

archivebox/config/paths.py

@@ -19,7 +19,7 @@ if TYPE_CHECKING:
 #############################################################################################
 
 PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent  # archivebox source code dir
-DATA_DIR: Path = Path(os.environ.get("DATA_DIR", os.getcwd())).resolve()  # archivebox user data dir
+DATA_DIR: Path = Path(os.getcwd()).resolve()  # archivebox user data dir
 
 
 def _env_path(key: str, default: Path) -> Path:
@@ -29,8 +29,8 @@ def _env_path(key: str, default: Path) -> Path:
     return path.resolve()
 
 
-ARCHIVE_DIR: Path = _env_path("ARCHIVE_DIR", DATA_DIR / "archive")  # archivebox snapshot data dir
-USERS_DIR: Path = _env_path("USERS_DIR", ARCHIVE_DIR / "users")  # archivebox user-scoped crawl/snapshot data dir
+ARCHIVE_DIR: Path = DATA_DIR / "archive"  # archivebox snapshot data dir
+USERS_DIR: Path = ARCHIVE_DIR / "users"  # archivebox user-scoped crawl/snapshot data dir
 
 IN_DOCKER = os.environ.get("IN_DOCKER", False) in ("1", "true", "True", "TRUE", "yes")
 
@@ -290,20 +290,20 @@ def get_data_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_mount": os.path.ismount(DATABASE_FILE.resolve()),
             },
             "ARCHIVE_DIR": {
-                "path": config.ARCHIVE_DIR.resolve(),
+                "path": CONSTANTS.ARCHIVE_DIR.resolve(),
                 "enabled": True,
-                "is_valid": os.path.isdir(config.ARCHIVE_DIR)
-                and os.access(config.ARCHIVE_DIR, os.R_OK)
-                and os.access(config.ARCHIVE_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.ARCHIVE_DIR.resolve()),
+                "is_valid": os.path.isdir(CONSTANTS.ARCHIVE_DIR)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.R_OK)
+                and os.access(CONSTANTS.ARCHIVE_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.ARCHIVE_DIR.resolve()),
             },
             "USERS_DIR": {
-                "path": config.USERS_DIR.resolve(),
-                "enabled": os.path.isdir(config.USERS_DIR),
-                "is_valid": os.path.isdir(config.USERS_DIR)
-                and os.access(config.USERS_DIR, os.R_OK)
-                and os.access(config.USERS_DIR, os.W_OK),
-                "is_mount": os.path.ismount(config.USERS_DIR.resolve()),
+                "path": CONSTANTS.USERS_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.USERS_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.USERS_DIR)
+                and os.access(CONSTANTS.USERS_DIR, os.R_OK)
+                and os.access(CONSTANTS.USERS_DIR, os.W_OK),
+                "is_mount": os.path.ismount(CONSTANTS.USERS_DIR.resolve()),
             },
             "SOURCES_DIR": {
                 "path": CONSTANTS.SOURCES_DIR.resolve(),
@@ -366,9 +366,9 @@ def get_code_locations(config: "ArchiveBoxConfig | None" = None, **config_kwargs
                 "is_valid": os.access(CONSTANTS.STATIC_DIR, os.R_OK) and os.access(CONSTANTS.STATIC_DIR, os.X_OK),  # read + list
             },
             "CUSTOM_TEMPLATES_DIR": {
-                "path": config.CUSTOM_TEMPLATES_DIR.resolve(),
-                "enabled": os.path.isdir(config.CUSTOM_TEMPLATES_DIR),
-                "is_valid": os.path.isdir(config.CUSTOM_TEMPLATES_DIR) and os.access(config.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
+                "path": CONSTANTS.CUSTOM_TEMPLATES_DIR.resolve(),
+                "enabled": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR),
+                "is_valid": os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK),  # read
             },
             "USER_PLUGINS_DIR": {
                 "path": CONSTANTS.USER_PLUGINS_DIR.resolve(),

archivebox/config/permissions.py

@@ -15,8 +15,6 @@ from contextlib import contextmanager
 #############################################################################################
 
 DATA_DIR = Path(os.getcwd())
-if os.environ.get("DATA_DIR") and Path(os.environ["DATA_DIR"]).resolve() != DATA_DIR.resolve():
-    raise SystemExit(f"[X] DATA_DIR={os.environ['DATA_DIR']} must equal cwd={DATA_DIR}; cd into the data dir before running archivebox")
 
 try:
     DATA_DIR_STAT = DATA_DIR.stat()

archivebox/config/views.py

@@ -24,7 +24,7 @@ INSTALLED_BINARIES_BASE_URL = "/admin/machine/binary/"
 
 
 def is_superuser(request: HttpRequest) -> bool:
-    return bool(getattr(request.user, "is_superuser", False))
+    return bool(request.user.is_superuser)
 
 
 def format_parsed_datetime(value: object) -> str:
@@ -44,7 +44,7 @@ def get_installed_binary_change_url(name: str, binary: Binary | None) -> str | N
     if binary is None or not binary.id:
         return None
 
-    base_url = getattr(binary, "admin_change_url", None) or f"{INSTALLED_BINARIES_BASE_URL}{binary.id}/change/"
+    base_url = binary.admin_change_url
     changelist_filters = urlencode({"q": name})
     return f"{base_url}?{urlencode({'_changelist_filters': changelist_filters})}"
 
@@ -158,7 +158,7 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
     for name in all_binary_names:
         binary = db_binaries.get(name)
-        binary_is_valid = bool(binary and getattr(binary, "is_valid", getattr(binary, "abspath", None)))
+        binary_is_valid = bool(binary and binary.is_valid)
 
         rows["Binary Name"].append(ItemLink(name, key=name))
 
@@ -185,9 +185,9 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         "youtube-dl": "yt-dlp",
     }.get(key, key)
     db_binary = get_db_binaries_by_name().get(key)
-    binary_is_valid = bool(db_binary and getattr(db_binary, "is_valid", getattr(db_binary, "abspath", None)))
+    binary_is_valid = bool(db_binary and db_binary.is_valid)
     if binary_is_valid:
-        binary_data = db_binary.to_json() if hasattr(db_binary, "to_json") else db_binary.__dict__
+        binary_data = db_binary.to_json()
         section: SectionData = {
             "name": key,
             "description": mark_safe(render_binary_detail_description(key, binary_data, db_binary)),

archivebox/core/admin_archiveresults.py

@@ -19,8 +19,6 @@ from django.urls import reverse, resolve
 from django.utils import timezone
 from django.utils.text import smart_split
 
-from archivebox.config import DATA_DIR
-from archivebox.config.common import get_config
 from archivebox.misc.paginators import AcceleratedPaginator
 from archivebox.base_models.admin import BaseModelAdmin
 from archivebox.plugins.discovery import get_plugin_icon
@@ -35,7 +33,7 @@ from archivebox.core.models import ArchiveResult, Snapshot
 
 def _get_replay_source_url(result: ArchiveResult) -> str:
     process = result.process_record
-    return str(getattr(process, "url", None) or result.snapshot.url or "")
+    return str((process.url if process else None) or result.snapshot.url or "")
 
 
 def build_abx_dl_display_command(result: ArchiveResult) -> str:
@@ -51,8 +49,8 @@ def build_abx_dl_display_command(result: ArchiveResult) -> str:
 
 def build_abx_dl_replay_command(result: ArchiveResult, config=None) -> str:
     display_command = build_abx_dl_display_command(result)
-    process = getattr(result, "process", None)
-    env_items = env_to_shell_exports(getattr(process, "env", None) or {})
+    process = result.process
+    env_items = env_to_shell_exports(process.env if process else {})
     if config is not None:
         result.snapshot._runtime_config = config
     snapshot_dir = shlex.quote(str(result.pwd or result.snapshot_dir))
@@ -164,8 +162,8 @@ def render_archiveresults_list(archiveresults_qs, limit=50, config=None):
         cmd_attr = html.escape(replay_cmd, quote=True)
 
         # Build output link - use embed_path() which checks output_files first
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        embed_path = result.embed_path()
+        snapshot_id = str(result.snapshot_id)
         if embed_path and result.status == "succeeded":
             output_link = build_snapshot_url(snapshot_id, embed_path, config=config)
         else:
@@ -354,8 +352,7 @@ class ArchiveResultInline(admin.TabularInline):
     def get_formset(self, request, obj=None, **kwargs):
         formset = super().get_formset(request, obj, **kwargs)
         snapshot = self.get_parent_object_from_request(request)
-        form_class = getattr(formset, "form", None)
-        base_fields = getattr(form_class, "base_fields", {})
+        base_fields = formset.form.base_fields
         snapshot_output_dir = str(snapshot.output_dir) if snapshot else ""
 
         # import ipdb; ipdb.set_trace()
@@ -491,12 +488,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         return super().change_view(request, object_id, form_url, extra_context)
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
         self.list_per_page = request.archivebox_config.SNAPSHOTS_PER_PAGE
         try:
@@ -573,20 +568,20 @@ class ArchiveResultAdmin(BaseModelAdmin):
         return queryset.filter(reduce(and_, filters)).distinct(), True
 
     def get_snapshot_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(result.snapshot_id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(result.snapshot_id), request=request, config=request.archivebox_config)
 
     def get_output_view_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
-        output_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        config = request.archivebox_config
+        output_path = result.embed_path()
         if not output_path:
             output_path = result.plugin or ""
         return build_snapshot_url(str(result.snapshot_id), output_path, request=request, config=config)
 
     def get_output_files_url(self, result: ArchiveResult) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(result.snapshot_id), result.plugin, request=request, config=request.archivebox_config)}/?files=1"
 
     def get_output_zip_url(self, result: ArchiveResult) -> str:
         return f"{self.get_output_files_url(result)}&download=zip"
@@ -612,10 +607,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
     )
     def snapshot_info(self, result):
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         return format_html(
             '<a href="{}"><b><code>[{}]</code></b> &nbsp; {} &nbsp; {}</a><br/>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
             snapshot_id[:8],
             result.snapshot.bookmarked_at.strftime("%Y-%m-%d %H:%M"),
             result.snapshot.url[:128],
@@ -687,9 +682,9 @@ class ArchiveResultAdmin(BaseModelAdmin):
 
     @admin.display(description="Command")
     def cmd_str(self, result):
-        request = getattr(self, "request", None)
+        request = self.request
         display_cmd = build_abx_dl_display_command(result)
-        replay_cmd = build_abx_dl_replay_command(result, config=getattr(request, "archivebox_config", None))
+        replay_cmd = build_abx_dl_replay_command(result, config=request.archivebox_config)
         return format_html(
             """
             <div style="position: relative; width: 100%; max-width: 100%; overflow: hidden; box-sizing: border-box;">
@@ -710,10 +705,10 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_display(self, result):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         # Determine output link path - use embed_path() which checks output_files
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else None
+        embed_path = result.embed_path()
         output_path = embed_path if (result.status == "succeeded" and embed_path) else "index.html"
         snapshot_id = str(result.snapshot_id)
         return format_html(
@@ -728,12 +723,12 @@ class ArchiveResultAdmin(BaseModelAdmin):
         if not output_text:
             return "-"
 
-        request = getattr(self, "request", None)
-        live_path = result.embed_path() if hasattr(result, "embed_path") else None
+        request = self.request
+        live_path = result.embed_path()
         if live_path:
             return format_html(
                 '<a href="{}" title="{}"><code>{}</code></a>',
-                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=getattr(request, "archivebox_config", None)),
+                build_snapshot_url(str(result.snapshot_id), live_path, request=request, config=request.archivebox_config),
                 output_text,
                 output_text,
             )
@@ -785,18 +780,18 @@ class ArchiveResultAdmin(BaseModelAdmin):
         )
 
     def output_summary(self, result):
-        snapshot_dir = Path(DATA_DIR) / str(result.pwd).split("data/", 1)[-1]
+        snapshot_dir = Path(result.snapshot.output_dir)
         output_html = format_html(
             '<pre style="display: inline-block">{}</pre><br/>',
             result.output_str_for_display(),
         )
         snapshot_id = str(result.snapshot_id)
-        request = getattr(self, "request", None)
+        request = self.request
         output_html += format_html(
             '<a href="{}#all">See result files ...</a><br/><pre><code>',
-            build_snapshot_url(snapshot_id, "index.html", request=request, config=getattr(request, "archivebox_config", None)),
+            build_snapshot_url(snapshot_id, "index.html", request=request, config=request.archivebox_config),
         )
-        embed_path = result.embed_path() if hasattr(result, "embed_path") else ""
+        embed_path = result.embed_path() or ""
         path_from_embed = snapshot_dir / (embed_path or "")
         output_html += format_html(
             '<i style="padding: 1px">{}</i><b style="padding-right: 20px">/</b><i>{}</i><br/><hr/>',

archivebox/core/admin_snapshots.py

@@ -11,7 +11,7 @@ from django.http import JsonResponse, HttpResponseBadRequest, HttpResponseNotAll
 from django.utils import timezone
 from django.utils.html import format_html, format_html_join
 from django.utils.safestring import mark_safe
-from django.db.models import Q, Count, Exists, OuterRef, Prefetch
+from django.db.models import Q, Count, Exists, F, OuterRef, Prefetch
 from django import forms
 from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
@@ -190,6 +190,7 @@ class SnapshotSizeListFilter(admin.SimpleListFilter):
 class SnapshotResultHealthListFilter(admin.SimpleListFilter):
     title = "ArchiveResult status"
     parameter_name = "archiveresult_status"
+    SNAPSHOT_FIRST_VALUES = {"succeeded"}
 
     def lookups(self, request, model_admin):
         return (
@@ -199,7 +200,6 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
             ("failed", ">50% failed"),
             ("running", ">50% running"),
             ("pending", ">50% queued"),
-            ("backoff", ">50% waiting to retry"),
             ("noresults", ">50% noresults"),
         )
 
@@ -215,27 +215,64 @@ class SnapshotResultHealthListFilter(admin.SimpleListFilter):
                 "succeeded": ArchiveResult.StatusChoices.SUCCEEDED,
                 "failed": ArchiveResult.StatusChoices.FAILED,
                 "running": ArchiveResult.StatusChoices.STARTED,
-                "pending": ArchiveResult.StatusChoices.QUEUED,
+                "pending": (ArchiveResult.StatusChoices.QUEUED, ArchiveResult.StatusChoices.BACKOFF),
                 "backoff": ArchiveResult.StatusChoices.BACKOFF,
                 "noresults": ArchiveResult.StatusChoices.NORESULTS,
             }
             if value in status_by_value:
-                # Start from ArchiveResult.status for every majority-status filter.
-                # The (status, snapshot_id) index keeps this plan stable regardless
-                # of which status is most common in a user's collection.
-                return queryset.filter(pk__in=ArchiveResult.snapshot_ids_with_majority_status(status_by_value[value]))
+                status = status_by_value[value]
+                if value in self.SNAPSHOT_FIRST_VALUES:
+                    # "succeeded" is overwhelmingly common in large collections.
+                    # Scan Snapshots in admin order and do indexed per-snapshot
+                    # probes so page 1 can stop after list_per_page matches.
+                    queryset = queryset.alias(
+                        total_results=ArchiveResult.snapshot_count_expr(),
+                        matching_results=ArchiveResult.snapshot_count_expr(status=status),
+                    ).filter(matching_results__gt=F("total_results") / 2)
+                    queryset._archivebox_count_hint = "model_estimate"
+                    queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+                    return queryset
+
+                # Rare statuses are faster status-first: use the
+                # (status, snapshot_id) index to find candidate snapshots.
+                snapshot_ids = ArchiveResult.cached_snapshot_ids_with_majority_status(status)
+                queryset = queryset.filter(pk__in=snapshot_ids)
+                queryset._archivebox_count_hint = len(snapshot_ids)
+                queryset.query._archivebox_count_hint = queryset._archivebox_count_hint
+                return queryset
         return queryset
 
 
 class SnapshotChangeList(SearchResultsChangeList):
     def __init__(self, request, *args, **kwargs):
         super().__init__(request, *args, **kwargs)
-        resolver_name = getattr(getattr(request, "resolver_match", None), "url_name", "")
+        resolver_name = request.resolver_match.url_name
         self.embedded_changelist = request.GET.get("_embedded") == "crawl"
         self.snapshot_is_grid_view = not self.embedded_changelist and (
             resolver_name == "grid" or request.path.rstrip("/").endswith("/grid")
         )
 
+    def _attach_archiveresult_summaries(self):
+        snapshot_ids = [obj.pk for obj in self.result_list]
+        if not snapshot_ids:
+            return
+
+        results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
+        seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
+        rows = (
+            ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
+            .order_by("snapshot_id", "plugin")
+            .values_list("snapshot_id", "plugin", "status", "output_size")
+        )
+        for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
+            if plugin in seen_plugins[snapshot_id]:
+                continue
+            seen_plugins[snapshot_id].add(plugin)
+            results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
+
+        for obj in self.result_list:
+            obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+
     def get_results(self, request):
         super().get_results(request)
         if request.GET.get("_embedded") == "crawl":
@@ -247,24 +284,7 @@ class SnapshotChangeList(SearchResultsChangeList):
                 self.list_per_page,
             ).count
         self.show_full_result_count = True
-
-        snapshot_ids = [obj.pk for obj in self.result_list]
-        if snapshot_ids:
-            results_by_snapshot = {snapshot_id: [] for snapshot_id in snapshot_ids}
-            seen_plugins = {snapshot_id: set() for snapshot_id in snapshot_ids}
-            rows = (
-                ArchiveResult.objects.filter(snapshot_id__in=snapshot_ids, status=ArchiveResult.StatusChoices.SUCCEEDED, output_size__gt=0)
-                .order_by("snapshot_id", "plugin")
-                .values_list("snapshot_id", "plugin", "status", "output_size")
-            )
-            for snapshot_id, plugin, status, output_size in rows.iterator(chunk_size=1000):
-                if plugin in seen_plugins[snapshot_id]:
-                    continue
-                seen_plugins[snapshot_id].add(plugin)
-                results_by_snapshot[snapshot_id].append(SimpleNamespace(plugin=plugin, status=status, output_size=output_size))
-
-            for obj in self.result_list:
-                obj.__dict__["_admin_archiveresults"] = results_by_snapshot[obj.pk]
+        self._attach_archiveresult_summaries()
 
 
 class SnapshotAdminForm(forms.ModelForm):
@@ -309,9 +329,7 @@ class SnapshotAdminForm(forms.ModelForm):
         # Handle tags_editor field
         if commit:
             instance.save()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get("tags_editor", "")
@@ -475,7 +493,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().get_ordering(request)
 
     def change_view(self, request, object_id, form_url="", extra_context=None):
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
+        self.request = request
         extra_context = extra_context or {}
         extra_context["CONFIG"] = request.archivebox_config
         snapshot = self.get_object(request, object_id)
@@ -490,7 +508,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     def changelist_view(self, request, extra_context=None):
         self.request = request
-        request.archivebox_config = getattr(request, "archivebox_config", None) or get_config()
         saved_list_per_page = self.list_per_page
         embedded_changelist = request.GET.get("_embedded") == "crawl"
         if embedded_changelist:
@@ -526,12 +543,12 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return super().lookup_allowed(lookup, value, request=request)
 
     def get_snapshot_view_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return build_snapshot_url(str(obj.id), request=request, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)
 
     def get_snapshot_files_url(self, obj: Snapshot) -> str:
-        request = getattr(self, "request", None)
-        return f"{build_snapshot_url(str(obj.id), request=request, config=getattr(request, 'archivebox_config', None))}/?files=1"
+        request = self.request
+        return f"{build_snapshot_url(str(obj.id), request=request, config=request.archivebox_config)}/?files=1"
 
     def get_snapshot_zip_url(self, obj: Snapshot) -> str:
         return f"{self.get_snapshot_files_url(obj)}&download=zip"
@@ -636,7 +653,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         ordering_fields = self._get_ordering_fields(request)
         needs_files_sort = "files" in ordering_fields
         needs_tags_sort = "tags_inline" in ordering_fields
-        is_change_view = getattr(getattr(request, "resolver_match", None), "url_name", "") == "core_snapshot_change"
+        is_change_view = request.resolver_match.url_name == "core_snapshot_change"
         prefetches = ["tags"]
         if is_change_view:
             prefetches.append(
@@ -688,7 +705,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="👁", ordering="permissions")
     def permissions_badge(self, obj):
-        permissions = getattr(obj, "snapshot_permissions", None)
+        permissions = obj.__dict__.get("snapshot_permissions")
         if permissions is None:
             if obj.permissions:
                 permissions = obj.permissions
@@ -858,8 +875,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def status_info(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         favicon_url = build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)
         return format_html(
             """
@@ -876,16 +893,16 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Archive Results")
     def archiveresults_list(self, obj):
-        request = getattr(self, "request", None)
-        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=getattr(request, "archivebox_config", None))
+        request = self.request
+        return render_archiveresults_list(obj.archiveresult_set.all(), limit=8, config=request.archivebox_config)
 
     @admin.display(
         description="Title",
         ordering="title",
     )
     def title_str(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         title_raw = (obj.title or "").strip()
         url_raw = (obj.url or "").strip()
         title_normalized = title_raw.lower()
@@ -938,8 +955,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         return mark_safe(f'<span class="tags-inline-editor">{tags_html}</span>')
 
     def _get_preview_data(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         results = self._get_prefetched_results(obj)
         if results is not None:
             has_screenshot = any(r.plugin == "screenshot" for r in results)
@@ -991,8 +1008,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         if not preview:
             return ""
 
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         favicon_url = build_snapshot_url(str(obj.id), "favicon/favicon.ico", request=request, config=config)
         fallback_list = ",".join([build_snapshot_url(str(obj.id), "favicon.ico", request=request, config=config)])
         onerror_js = (
@@ -1030,8 +1047,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description=" ", empty_value="")
     def snapshot_summary(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         preview = self._get_preview_data(obj)
         stats = self._get_progress_stats(obj)
         archive_size = stats["output_size"] or 0
@@ -1095,8 +1112,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
         visible_results = sorted_results[:14]
         output = []
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         for result in visible_results:
             icon = mark_safe(get_plugin_icon(result.plugin))
             if not icon.strip():
@@ -1127,8 +1144,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         # ordering='archiveresult_count'
     )
     def size(self, obj):
-        request = getattr(self, "request", None)
-        config = getattr(request, "archivebox_config", None)
+        request = self.request
+        config = request.archivebox_config
         archive_size = self._get_progress_stats(obj)["output_size"] or 0
         if archive_size:
             size_txt = printable_filesize(archive_size)
@@ -1242,7 +1259,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         )
 
     def _get_progress_stats(self, obj):
-        cached_stats = getattr(obj, "_admin_progress_stats", None)
+        cached_stats = obj.__dict__.get("_admin_progress_stats")
         if cached_stats is not None:
             return cached_stats
 
@@ -1288,31 +1305,31 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
     def _get_prefetched_results(self, obj):
         if "_admin_archiveresults" in obj.__dict__:
             return obj.__dict__["_admin_archiveresults"]
-        if hasattr(obj, "_prefetched_objects_cache") and "archiveresult_set" in obj._prefetched_objects_cache:
+        if "archiveresult_set" in obj.__dict__.get("_prefetched_objects_cache", {}):
             return obj.archiveresult_set.all()
         return None
 
     def _get_expected_hook_total(self, obj) -> int:
         try:
-            request = getattr(self, "request", None)
-            if getattr(getattr(request, "resolver_match", None), "url_name", "") in {"core_snapshot_changelist", "core_snapshot_change"}:
+            request = self.request
+            if request.resolver_match.url_name in {"core_snapshot_changelist", "core_snapshot_change"}:
                 return 0
 
-            crawl = getattr(obj, "crawl", None)
-            snapshot_config = getattr(obj, "config", None) or {}
-            crawl_config = getattr(crawl, "config", None) or {}
+            crawl = obj.crawl
+            snapshot_config = obj.config or {}
+            crawl_config = crawl.config or {}
             has_scoped_config = bool(snapshot_config or crawl_config)
 
             if request is not None and not has_scoped_config:
-                cached_total = getattr(request, "archivebox_expected_snapshot_hook_total", None)
+                cached_total = request.__dict__.get("archivebox_expected_snapshot_hook_total")
                 if cached_total is None:
-                    config = getattr(request, "archivebox_config", None) or get_config()
+                    config = request.archivebox_config
                     cached_total = len(discover_hooks("Snapshot", config=config))
                     request.archivebox_expected_snapshot_hook_total = cached_total
                 return cached_total
 
             if request is not None:
-                scoped_cache = getattr(request, "archivebox_expected_snapshot_hook_totals_by_scope", None)
+                scoped_cache = request.__dict__.get("archivebox_expected_snapshot_hook_totals_by_scope")
                 if scoped_cache is None:
                     scoped_cache = {}
                     request.archivebox_expected_snapshot_hook_totals_by_scope = scoped_cache
@@ -1332,8 +1349,9 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
             return 0
 
     def _get_prefetched_tags(self, obj):
-        if hasattr(obj, "_prefetched_objects_cache") and "tags" in obj._prefetched_objects_cache:
-            return list(obj._prefetched_objects_cache["tags"])
+        prefetched_cache = obj.__dict__.get("_prefetched_objects_cache", {})
+        if "tags" in prefetched_cache:
+            return list(prefetched_cache["tags"])
         return None
 
     def _get_ordering_fields(self, request):

archivebox/core/admin_tags.py

@@ -231,7 +231,7 @@ class TagAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshots", ordering="num_snapshots")
     def num_snapshots(self, tag: Tag):
-        count = getattr(tag, "num_snapshots", None)
+        count = tag.__dict__.get("num_snapshots")
         if count is None:
             count = tag.snapshot_set.count()
         return format_html(

archivebox/core/admin_users.py

@@ -47,7 +47,7 @@ class CustomUserAdmin(UserAdmin):
 
     def snapshot_count_badge(self, obj):
         snapshots_url = f"/admin/core/snapshot/?created_by__id__exact={obj.pk}"
-        snapshot_count = getattr(obj, "snapshot_count", 0)
+        snapshot_count = obj.__dict__.get("snapshot_count", 0)
         snapshot_label = "snapshot" if snapshot_count == 1 else "snapshots"
         return format_html(
             (

archivebox/core/forms.py

@@ -31,6 +31,8 @@ DEPTH_CHOICES = (
 
 
 class AddLinkForm(PluginConfigFormMixin, forms.Form):
+    allow_crawl_execution_config_fields = False
+
     # Basic fields
     url = forms.CharField(
         label="URLs",

archivebox/core/middleware.py

@@ -119,7 +119,7 @@ def CacheControlMiddleware(get_response):
 
         if "/archive/" in request.path or "/static/" in request.path or snapshot_path_re.match(request.path):
             if not response.get("Cache-Control"):
-                config = getattr(request, "archivebox_config", None)
+                config = request.__dict__.get("archivebox_config")
                 if config is None:
                     config = get_config(resolve_plugins=False)
                     request.archivebox_config = config
@@ -136,7 +136,7 @@ def ServerSecurityModeMiddleware(get_response):
     allowed_methods = {"GET", "HEAD", "OPTIONS"}
 
     def middleware(request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
@@ -169,7 +169,7 @@ def HostRoutingMiddleware(get_response):
             return get_response(request)
 
         request_host = (request.get_host() or "").lower()
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config
@@ -323,7 +323,7 @@ class ReverseProxyAuthMiddleware(RemoteUserMiddleware):
     header = "HTTP_REMOTE_USER"
 
     def process_request(self, request):
-        config = getattr(request, "archivebox_config", None)
+        config = request.__dict__.get("archivebox_config")
         if config is None:
             config = get_config(resolve_plugins=False)
             request.archivebox_config = config

archivebox/core/migrations/0048_remove_archiveresult_config.py

@@ -0,0 +1,14 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0047_archiveresult_status_snapshot_index"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="archiveresult",
+            name="config",
+        ),
+    ]

archivebox/core/models.py

@@ -14,7 +14,7 @@ from urllib.parse import urlparse
 from statemachine import State, registry
 
 from django.db import models, transaction
-from django.db.models import Case, Q, QuerySet, Sum, Value, When
+from django.db.models import Case, F, Q, QuerySet, Sum, Value, When
 from django.db.models.functions import Coalesce, Concat
 from django.db.models.fields.json import KT
 from django.utils.functional import cached_property
@@ -63,6 +63,13 @@ if TYPE_CHECKING:
     from archivebox.config.common import ArchiveBoxBaseConfig
 
 
+class UngroupedSubquery(models.Subquery):
+    """Scalar subquery that should not be copied into the outer GROUP BY."""
+
+    def get_group_by_cols(self):
+        return []
+
+
 class Tag(ModelWithUUID):
     # Keep AutoField for compatibility with main branch migrations
     # Don't use UUIDField here - requires complex FK transformation
@@ -203,7 +210,7 @@ class SnapshotQuerySet(models.QuerySet):
                 offset += chunk_size
             return
 
-        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if getattr(field, "unique", False))}
+        unique_field_names = {pk_field, *(field.name for field in self.model._meta.fields if field.unique)}
         if not any(field_name in unique_field_names for field_name in ordered_field_names):
             offset = 0
             while True:
@@ -457,6 +464,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     objects = SnapshotManager()
     archiveresult_set: models.Manager["ArchiveResult"]
 
+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> "SnapshotMachine": ...
+
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
@@ -659,7 +671,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             self.sm.seal()
 
     def get_delete_after_config_value(self):
-        return get_config(snapshot=self).DELETE_AFTER
+        from archivebox.config.common import resolve_delete_after_config_value
+
+        return resolve_delete_after_config_value(self.config, self.crawl.config)
 
     @classmethod
     def missing_delete_at_candidates(cls):
@@ -818,7 +832,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         # work and crawl maintenance so SQLite commits before touching the disk.
         transaction.on_commit(finish_snapshot_save)
 
-        migration_cleanup = getattr(self, "_pending_fs_migration_cleanup", None)
+        migration_cleanup = self.__dict__.get("_pending_fs_migration_cleanup")
         if migration_cleanup:
             old_dir, new_dir = migration_cleanup
             transaction.on_commit(lambda: self._cleanup_old_migration_dir(old_dir, new_dir))
@@ -890,9 +904,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         runtime_config = config or get_config()
 
         if source_dir and current == target:
-            current_dir = self.get_storage_path_for_version(target, config=runtime_config)
+            current_dir = self.get_storage_path_for_version(target)
             cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir, target_dir=current_dir)
-            crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+            crawl_dir = self.crawl.output_dir
             old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
             if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
                 crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -932,9 +946,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     def _fs_migrate_from_0_9_0_to_0_9_4(self, source_dir: Path | None = None, config: "ArchiveBoxBaseConfig | None" = None):
         runtime_config = config or get_config()
-        target_dir = self.get_storage_path_for_version("0.9.4", config=runtime_config)
+        target_dir = self.get_storage_path_for_version("0.9.4")
         cleanup = self._fs_migrate_legacy_to_0_9_0(source_dir=source_dir or self.output_dir, target_dir=target_dir, config=runtime_config)
-        crawl_dir = self.crawl.output_dir_for_config(runtime_config)
+        crawl_dir = self.crawl.output_dir
         old_crawl_dir = crawl_dir.with_name(str(uuid.UUID(hex=self.crawl.id.hex)))
         if old_crawl_dir.exists() and not crawl_dir.exists() and not old_crawl_dir.is_symlink():
             crawl_dir.parent.mkdir(parents=True, exist_ok=True)
@@ -956,8 +970,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         import filecmp
         import shutil
 
-        old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0", config=config)
-        new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0", config=config)
+        old_dir = Path(source_dir) if source_dir else self.get_storage_path_for_version("0.8.0")
+        new_dir = Path(target_dir) if target_dir else self.get_storage_path_for_version("0.9.0")
 
         if old_dir == new_dir:
             return None
@@ -1087,7 +1101,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         except Exception:
             return "unknown"
 
-    def get_storage_path_for_version(self, version: str, config: "ArchiveBoxBaseConfig | None" = None) -> Path:
+    def get_storage_path_for_version(self, version: str) -> Path:
         """
         Calculate storage path for specific filesystem version.
         Centralizes path logic so it's reusable.
@@ -1095,10 +1109,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         0.7.x/0.8.x: archive/{timestamp}
         0.9.x: archive/users/{username}/snapshots/YYYYMMDD/{domain}/{uuid}/
         """
-        runtime_config = config or get_config()
-
         if version in ("0.7.0", "0.8.0"):
-            return runtime_config.ARCHIVE_DIR / self.timestamp
+            return CONSTANTS.ARCHIVE_DIR / self.timestamp
 
         elif version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
             username = self.created_by.username
@@ -1108,10 +1120,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
             domain = self.extract_domain_from_url(self.url)
 
-            return runtime_config.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
+            return CONSTANTS.USERS_DIR / username / CONSTANTS.SNAPSHOTS_DIR_NAME / date_str / domain / str(self.id)
         else:
             # Unknown version - use current
-            return self.get_storage_path_for_version(self._fs_current_version(), config=runtime_config)
+            return self.get_storage_path_for_version(self._fs_current_version())
 
     # =========================================================================
     # Loading and Creation from Filesystem (Used by archivebox update ONLY)
@@ -1702,7 +1714,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                     f.write(json.dumps(process.to_json()) + "\n")
 
                 # Write ArchiveResult record
-                f.write(json.dumps(ar.to_json()) + "\n")
+                f.write(json.dumps(ar.to_json(snapshot_output_dir=output_dir)) + "\n")
         os.replace(tmp_index_path, index_path)
 
     def read_index_jsonl(self, output_dir: Path | None = None) -> dict:
@@ -1978,7 +1990,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
     @admin.display(description="Tags")
     def tags_str(self, nocache=True) -> str | None:
         calc_tags_str = lambda: ",".join(sorted(tag.name for tag in self.tags.all()))
-        prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+        prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
         if "tags" in prefetched_cache:
             return calc_tags_str()
         cache_key = f"{self.pk}-tags"
@@ -1988,12 +2000,12 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         """Generate HTML icons showing which extractor plugins have succeeded for this snapshot"""
         from django.utils.html import format_html
 
-        compact_icons = getattr(self, "_icons_compact", False)
+        compact_icons = self.__dict__.get("_icons_compact", False)
         cache_key = f"result_icons:{self.pk}:{'compact' if compact_icons else 'full'}:{(self.downloaded_at or self.modified_at or self.created_at or self.bookmarked_at).timestamp()}"
 
         def calc_icons():
             if compact_icons and self.status == self.StatusChoices.STARTED:
-                progress_stats = getattr(self, "_icons_progress_stats", None) or self.get_progress_stats()
+                progress_stats = self.__dict__.get("_icons_progress_stats") or self.get_progress_stats()
                 total = int(progress_stats.get("total") or 0)
                 succeeded = int(progress_stats.get("succeeded") or 0)
                 failed = int(progress_stats.get("failed") or 0)
@@ -2025,8 +2037,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                     running,
                 )
 
-            precomputed_archive_results = getattr(self, "_icons_archive_results", None)
-            prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+            precomputed_archive_results = self.__dict__.get("_icons_archive_results")
+            prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
             if precomputed_archive_results is not None and compact_icons:
                 archive_results = {plugin: True for plugin in precomputed_archive_results}
             elif "archiveresult_set" in prefetched_cache:
@@ -2049,7 +2061,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
             output_template = '<a href="/{}/{}" class="exists-{}" title="{}">{}</a>'
 
             # Get all plugins from hooks system (sorted by numeric prefix)
-            all_plugins = getattr(self, "_icons_plugin_names", None)
+            all_plugins = self.__dict__.get("_icons_plugin_names")
             if all_plugins is None and not compact_icons:
                 all_plugins = [get_plugin_name(e) for e in get_plugins()]
             elif all_plugins is None:
@@ -2200,8 +2212,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         """The filesystem path to the snapshot's output directory."""
         import os
 
-        runtime_config = getattr(self, "_runtime_config", None) or get_config()
-        current_path = self.get_storage_path_for_version(self.fs_version, config=runtime_config)
+        current_path = self.get_storage_path_for_version(self.fs_version)
 
         if current_path.exists():
             return current_path
@@ -2212,7 +2223,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
                 return hyphen_path
 
         # Check for backwards-compat symlink
-        old_path = runtime_config.ARCHIVE_DIR / self.timestamp
+        old_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
         if old_path.is_symlink():
             link_target = Path(os.readlink(old_path))
             return (old_path.parent / link_target).resolve() if not link_target.is_absolute() else link_target.resolve()
@@ -2225,7 +2236,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
         """Ensure the legacy archive/<timestamp> path resolves to this snapshot."""
         import os
 
-        legacy_path = get_config().ARCHIVE_DIR / self.timestamp
+        legacy_path = CONSTANTS.ARCHIVE_DIR / self.timestamp
         target = Path(self.get_storage_path_for_version(self._fs_current_version()))
 
         if target == legacy_path:
@@ -2301,8 +2312,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         if self.fs_version in ("0.9.0", "0.9.1", "0.9.2", "0.9.3", "0.9.4", "1.0.0"):
             username = "web"
-            crawl = getattr(self, "crawl", None)
-            if crawl and getattr(crawl, "created_by_id", None):
+            crawl = self.crawl if self.crawl_id else None
+            if crawl and crawl.created_by_id:
                 username = crawl.created_by.username
             if username == "system":
                 username = "web"
@@ -2326,7 +2337,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
         output_dir = Path(self.output_dir).resolve()
         try:
-            rel_users_path = output_dir.relative_to(get_config().USERS_DIR)
+            rel_users_path = output_dir.relative_to(CONSTANTS.USERS_DIR)
         except Exception:
             rel_users_path = None
 
@@ -2836,7 +2847,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @cached_property
     def is_archived(self) -> bool:
-        cached_is_archived = getattr(self, "_is_archived_cached", None)
+        cached_is_archived = self.__dict__.get("_is_archived_cached")
         if cached_is_archived is not None:
             return bool(cached_is_archived)
 
@@ -2890,10 +2901,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @cached_property
     def num_outputs(self) -> int:
-        if hasattr(self, "num_outputs_cached"):
-            return int(self.num_outputs_cached or 0)
+        if "num_outputs_cached" in self.__dict__:
+            return int(self.__dict__["num_outputs_cached"] or 0)
 
-        prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+        prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
         if "archiveresult_set" in prefetched_cache:
             return sum(1 for result in self.archiveresult_set.all() if result.status == "succeeded")
 
@@ -2901,10 +2912,10 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
 
     @cached_property
     def num_failures(self) -> int:
-        if hasattr(self, "num_failures_cached"):
-            return int(self.num_failures_cached or 0)
+        if "num_failures_cached" in self.__dict__:
+            return int(self.__dict__["num_failures_cached"] or 0)
 
-        prefetched_cache = getattr(self, "_prefetched_objects_cache", {})
+        prefetched_cache = self.__dict__.get("_prefetched_objects_cache", {})
         if "archiveresult_set" in prefetched_cache:
             return sum(1 for result in self.archiveresult_set.all() if result.status == "failed")
 
@@ -3458,7 +3469,7 @@ class SnapshotMachine(BaseStateMachine):
         # and the runner still needs to enqueue those child snapshots.
 
 
-class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithNotes):
+class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
     class StatusChoices(models.TextChoices):
         QUEUED = "queued", "Queued"
         STARTED = "started", "Started"
@@ -3523,7 +3534,17 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         qs = cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
         if status is not None:
             qs = qs.filter(status=status)
-        return qs.order_by().values("snapshot_id").annotate(count=models.Count("pk")).values("count")
+        return qs.order_by().values("snapshot_id").annotate(count=models.Count("*")).values("count")
+
+    @classmethod
+    def snapshot_half_count_subquery(cls, *, outer_ref: str = "snapshot_id") -> QuerySet:
+        return (
+            cls.objects.filter(snapshot_id=models.OuterRef(outer_ref))
+            .order_by()
+            .values("snapshot_id")
+            .annotate(half=models.Count("*") / models.Value(2))
+            .values("half")
+        )
 
     @classmethod
     def snapshot_count_expr(cls, *, status: str | None = None, outer_ref: str = "pk"):
@@ -3539,28 +3560,50 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         return {status: qs.filter(status=status).count() for status in (statuses or cls.StatusChoices.values)}
 
     @classmethod
-    def snapshot_ids_with_majority_status(cls, status: str) -> QuerySet:
+    def snapshot_ids_with_majority_status(cls, status: str | Iterable[str]) -> QuerySet:
         """Return Snapshot IDs where more than half of ArchiveResults have ``status``.
 
         Start from ArchiveResult.status for every majority-status filter. The
         ``(status, snapshot_id)`` index keeps the plan predictable even when a
         user's collection has an unusual status distribution.
         """
+        statuses = tuple(status) if not isinstance(status, str) else (status,)
+        total_half = UngroupedSubquery(cls.snapshot_half_count_subquery(outer_ref="snapshot_id"), output_field=models.IntegerField())
         return (
-            cls.objects.filter(status=status)
+            cls.objects.filter(status__in=statuses)
             .order_by()
             .values("snapshot_id")
             .annotate(
-                matching_results=models.Count("pk"),
-                total_results=models.Subquery(
-                    cls.snapshot_count_subquery(outer_ref="snapshot_id"),
-                    output_field=models.IntegerField(),
-                ),
+                matching_results=models.Count("*"),
+                total_half=total_half,
             )
-            .filter(matching_results__gt=models.F("total_results") / 2)
+            .filter(matching_results__gt=models.F("total_half"))
             .values("snapshot_id")
         )
 
+    @classmethod
+    def cached_snapshot_ids_with_majority_status(cls, status: str | Iterable[str], *, timeout: int = 60) -> tuple[str, ...]:
+        statuses = tuple(status) if not isinstance(status, str) else (status,)
+        cache_key = f"archivebox:archiveresult:majority_status:{':'.join(sorted(statuses))}"
+        cached_ids = cache.get(cache_key)
+        if cached_ids is not None:
+            return tuple(cached_ids)
+
+        snapshot_ids = tuple(
+            str(snapshot_id) for snapshot_id in cls.snapshot_ids_with_majority_status(statuses).values_list("snapshot_id", flat=True)
+        )
+        cache.set(cache_key, snapshot_ids, timeout=timeout)
+        return snapshot_ids
+
+    @classmethod
+    def clear_majority_status_cache(cls) -> None:
+        cache.delete_many(
+            [
+                *(f"archivebox:archiveresult:majority_status:{status}" for status in cls.StatusChoices.values),
+                f"archivebox:archiveresult:majority_status:{':'.join(sorted((cls.StatusChoices.BACKOFF, cls.StatusChoices.QUEUED)))}",
+            ],
+        )
+
     # UUID primary key (migrated from integer in 0029)
     id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -3609,7 +3652,6 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
-        ModelWithConfig.Meta,
         ModelWithNotes.Meta,
     ):
         app_label = "core"
@@ -3644,14 +3686,15 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         return "\n".join(self._format_output_line_for_display(line) for line in str(self.output_str or "").splitlines())
 
     def get_delete_after_config_value(self):
-        return get_config(archiveresult=self).DELETE_AFTER
+        snapshot = self.snapshot
+        from archivebox.config.common import resolve_delete_after_config_value
+
+        return resolve_delete_after_config_value(snapshot.config, snapshot.crawl.config)
 
     @classmethod
     def missing_delete_at_candidates(cls):
         return cls.objects.filter(delete_at__isnull=True).filter(
-            Q(config__has_key="DELETE_AFTER")
-            | Q(snapshot__config__has_key="DELETE_AFTER")
-            | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
+            Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
         )
 
     @property
@@ -3659,12 +3702,21 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         """Convenience property to access the user who created this archive result via its snapshot's crawl."""
         return self.snapshot.crawl.created_by
 
-    def to_json(self) -> dict:
+    def to_json(self, *, snapshot_output_dir: Path | None = None) -> dict:
         """
         Convert ArchiveResult model instance to a JSON-serializable dict.
         """
         from archivebox.config import VERSION
 
+        process = self.process_record
+        pwd = (
+            process.pwd
+            if process and process.pwd
+            else str((snapshot_output_dir / self.plugin) if snapshot_output_dir is not None else self.output_dir)
+        )
+        cmd = process.cmd if process else []
+        cmd_version = process.cmd_version if process else ""
+
         record = {
             "type": "ArchiveResult",
             "schema_version": VERSION,
@@ -3686,13 +3738,12 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             record["output_size"] = self.output_size
         if self.output_mimetypes:
             record["output_mimetypes"] = self.output_mimetypes
-        if self.pwd:
-            record["pwd"] = self.pwd
-        if self.cmd:
-            record["cmd"] = self.cmd
-        if self.cmd_version:
-            record["cmd_version"] = self.cmd_version
-        process = self.process_record
+        if pwd:
+            record["pwd"] = pwd
+        if cmd:
+            record["cmd"] = cmd
+        if cmd_version:
+            record["cmd_version"] = cmd_version
         if process:
             record["process_id"] = str(process.id)
         return record
@@ -3752,21 +3803,51 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
             or "snapshot_id" in update_fields
         )
         old_snapshot_id = None
+        old_output_size = 0
         if refresh_snapshot_size and not is_new:
-            old_snapshot_id = type(self).objects.filter(pk=self.pk).values_list("snapshot_id", flat=True).first()
-
-        update_fields = kwargs.get("update_fields")
-        if self.delete_at is None:
-            self.set_delete_at_from_config()
-            if self.delete_at is not None and update_fields is not None:
-                kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "delete_at"]))
+            old_values = type(self).objects.filter(pk=self.pk).values("snapshot_id", "output_size").first()
+            if old_values:
+                old_snapshot_id = old_values["snapshot_id"]
+                old_output_size = int(old_values["output_size"] or 0)
 
+        # ArchiveResult rows are updated on every plugin event. Resolving
+        # DELETE_AFTER here is deceptively expensive because the effective
+        # value lives on Snapshot/Crawl config, so a save of an already-loaded
+        # result can still materialize parent objects and parse config. The
+        # orchestrator owns the repair pass for these rows instead: it fills
+        # missing delete_at values from fresh Snapshot/Crawl config when the
+        # queue is idle, outside the hook-result write hot path.
         # Skip ModelWithOutputDir.save() to avoid creating index.json in plugin directories
         # Call the Django Model.save() directly instead
         models.Model.save(self, *args, **kwargs)
         if refresh_snapshot_size:
-            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, self.snapshot_id) if snapshot_id}
-            transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            current_snapshot_id = self.snapshot_id
+            snapshot_ids = {snapshot_id for snapshot_id in (old_snapshot_id, current_snapshot_id) if snapshot_id}
+            current_output_size = int(self.output_size or 0)
+            if len(snapshot_ids) > 1:
+                # Moving an ArchiveResult between Snapshots is rare and cannot
+                # be represented as a single delta on one parent row. Keep the
+                # conservative aggregate fallback for that shape.
+                transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes(snapshot_ids))
+            elif current_snapshot_id:
+                # Hook-result projection updates ArchiveResult rows at very
+                # high frequency during indexing. Re-aggregating every sibling
+                # row for the parent Snapshot on each save turns those short
+                # writes into a table-scan hot path. For the common case where
+                # the result stays attached to the same Snapshot, the persisted
+                # parent total is exactly the old total plus this row's size
+                # delta; F() keeps that update atomic with concurrent result
+                # saves for other plugins on the same Snapshot.
+                size_delta = current_output_size if is_new else current_output_size - old_output_size
+                if size_delta:
+                    transaction.on_commit(
+                        lambda: Snapshot.objects.filter(pk=current_snapshot_id).update(
+                            output_size=F("output_size") + size_delta,
+                            modified_at=timezone.now(),
+                        ),
+                    )
+        if is_new or update_fields is None or "status" in update_fields or "snapshot" in update_fields or "snapshot_id" in update_fields:
+            transaction.on_commit(type(self).clear_majority_status_cache)
 
         # if is_new:
         #     from archivebox.misc.logging_util import log_worker_event
@@ -3788,6 +3869,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
         deleted = super().delete(*args, **kwargs)
         if snapshot_id:
             transaction.on_commit(lambda: type(self).refresh_snapshot_output_sizes({snapshot_id}))
+            transaction.on_commit(type(self).clear_majority_status_cache)
         return deleted
 
     @staticmethod
@@ -4495,7 +4577,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, M
     def _url_passes_filters(self, url: str) -> bool:
         """Check if URL passes URL_ALLOWLIST and URL_DENYLIST config filters.
 
-        Uses proper config hierarchy: defaults -> file -> env -> machine -> user -> crawl -> snapshot
+        Uses the centralized config resolver so frozen crawl/snapshot values
+        and live Machine/Persona execution values apply in their scoped order.
         """
         return self.snapshot.crawl.url_passes_filters(url, snapshot=self.snapshot)
 

archivebox/core/settings.py

@@ -174,9 +174,9 @@ except ImportError:
 
 STATIC_URL = "/static/"
 TEMPLATES_DIR_NAME = "templates"
-CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONFIG.CUSTOM_TEMPLATES_DIR) and os.access(CONFIG.CUSTOM_TEMPLATES_DIR, os.R_OK)
+CUSTOM_TEMPLATES_ENABLED = os.path.isdir(CONSTANTS.CUSTOM_TEMPLATES_DIR) and os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK)
 STATICFILES_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / "static")] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'static')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -187,7 +187,7 @@ STATICFILES_DIRS = [
 ]
 
 TEMPLATE_DIRS = [
-    *([str(CONFIG.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
+    *([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
     # *[
     #     str(plugin_dir / 'templates')
     #     for plugin_dir in PLUGIN_DIRS.values()
@@ -334,7 +334,7 @@ STORAGES = {
         "BACKEND": "django.core.files.storage.FileSystemStorage",
         "OPTIONS": {
             "base_url": "/archive/",
-            "location": CONFIG.ARCHIVE_DIR,
+            "location": CONSTANTS.ARCHIVE_DIR,
         },
     },
     # "snapshots": {

archivebox/core/settings_logging.py

@@ -182,6 +182,7 @@ SETTINGS_LOGGING = {
             "formatter": "rich",
             "level": "DEBUG",
             "markup": False,
+            "enable_link_path": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
             "console": STDERR,
             "filters": ["noisyrequestsfilter", "daphneclosetimeout", "asynciocancelledshield", "stripansi"],
@@ -198,6 +199,7 @@ SETTINGS_LOGGING = {
         "outbound_webhooks": {
             "class": "rich.logging.RichHandler",
             "markup": False,
+            "enable_link_path": False,
             "rich_tracebacks": False,  # Use standard Python tracebacks (no frame/box)
             "formatter": "outbound_webhooks",
         },

archivebox/core/shutdown_util.py

@@ -134,7 +134,10 @@ def foreground_shutdown_signals(
         # signal for hard foreground-command shutdown. Server/update/run and
         # other non-interactive commands raise immediately so their finally
         # blocks can stop owned children without prompting.
-        if raise_on_first_signal or already_requested:
+        if already_requested:
+            os.write(sys.stdout.fileno(), f"\n[🛑] Got {sig.name} again, exiting immediately.\n".encode())
+            os._exit(130)
+        if raise_on_first_signal:
             raise KeyboardInterrupt
 
     try:
@@ -143,6 +146,8 @@ def foreground_shutdown_signals(
             signal.signal(sig, raise_keyboard_interrupt)
         yield state
     finally:
+        if state.signal_name and previous_active_state is not None and not previous_active_state.signal_name:
+            previous_active_state.signal_name = state.signal_name
         _active_shutdown_state = previous_active_state
         for sig, previous_handler in previous_handlers.items():
             signal.signal(sig, previous_handler)

archivebox/core/tag_util.py

@@ -246,7 +246,7 @@ def _build_snapshot_preview_map(
 
 
 def build_tag_card(tag: Tag, snapshot_previews: list[dict[str, Any]] | None = None) -> dict[str, Any]:
-    count = getattr(tag, "num_snapshots", None)
+    count = tag.__dict__.get("num_snapshots")
     if count is None:
         count = tag.snapshot_set.count()
     return {

archivebox/core/templatetags/core_tags.py

@@ -76,6 +76,12 @@ def _normalize_output_files(output_files: Any) -> dict[str, dict[str, Any]]:
     return {}
 
 
+def _snapshot_id(value: Any) -> Any:
+    from archivebox.core.models import Snapshot
+
+    return value.id if isinstance(value, Snapshot) else value
+
+
 def _coerce_output_file_size(value: Any) -> int | None:
     try:
         return max(int(value or 0), 0)
@@ -85,7 +91,7 @@ def _coerce_output_file_size(value: Any) -> int | None:
 
 def _count_media_files(result) -> int:
     try:
-        output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+        output_files = _normalize_output_files(result.output_files or {})
     except Exception:
         output_files = {}
 
@@ -121,7 +127,7 @@ def _list_media_files(result) -> list[dict]:
     except Exception:
         return media_files
 
-    output_files = _normalize_output_files(getattr(result, "output_files", None) or {})
+    output_files = _normalize_output_files(result.output_files or {})
     candidates: list[tuple[Path, int | None]] = []
     if output_files:
         for path, metadata in output_files.items():
@@ -300,7 +306,7 @@ def result_list(context, cl):
     """
     num_sorted_fields = 0
     request = context.get("request")
-    config = getattr(request, "archivebox_config", None) or context.get("CONFIG")
+    config = request.__dict__.get("archivebox_config") if request is not None else context.get("CONFIG")
     results = cl.result_list
     if config is not None:
         for obj in results:
@@ -443,7 +449,7 @@ def _unconfigured_banner_context(request) -> dict:
     # operator's clipboard already aligned with subdomain routing. The config
     # parser strips the leading ``*.`` so users can paste it verbatim.
     suggested_base_url = f"{scheme}://*.{canonical_host}" if canonical_host else ""
-    user = getattr(request, "user", None)
+    user = request.user
     is_superuser = bool(user and user.is_authenticated and user.is_superuser)
     machine_admin_url = ""
     if is_superuser:
@@ -496,19 +502,19 @@ def public_base_url(context) -> str:
 
 @register.simple_tag(takes_context=True)
 def snapshot_base_url(context, snapshot) -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
     return get_snapshot_base_url(str(snapshot_id), request=context.get("request"), config=context.get("CONFIG"))
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
     return build_snapshot_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
 
 
 @register.simple_tag(takes_context=True)
 def snapshot_preview_url(context, snapshot, path: str = "") -> str:
-    snapshot_id = getattr(snapshot, "id", snapshot)
+    snapshot_id = _snapshot_id(snapshot)
     return _build_snapshot_preview_url(str(snapshot_id), path, request=context.get("request"), config=context.get("CONFIG"))
 
 
@@ -538,16 +544,18 @@ def plugin_card(context, result) -> str:
         - output_path: Path to output relative to snapshot dir (from embed_path())
         - plugin: Plugin base name
     """
-    if result is None or not hasattr(result, "plugin"):
+    from archivebox.core.models import ArchiveResult
+
+    if result is None or not isinstance(result, ArchiveResult):
         return ""
 
     plugin = get_plugin_name(result.plugin)
     template_str = get_plugin_template(plugin, "card")
 
     # Use embed_path() for the display path
-    raw_output_path = result.embed_path() if hasattr(result, "embed_path") else ""
+    raw_output_path = result.embed_path() or ""
     output_url = build_snapshot_url(
-        str(getattr(result, "snapshot_id", "")),
+        str(result.snapshot_id),
         raw_output_path or "",
         request=context.get("request"),
         config=context.get("CONFIG"),
@@ -558,7 +566,7 @@ def plugin_card(context, result) -> str:
     media_file_count = _count_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else 0
     media_files = _list_media_files(result) if plugin_lower in ("ytdlp", "yt-dlp", "youtube-dl") else []
     if media_files:
-        snapshot_id = str(getattr(result, "snapshot_id", ""))
+        snapshot_id = str(result.snapshot_id)
         request = context.get("request")
         config = context.get("CONFIG")
         for item in media_files:
@@ -592,7 +600,7 @@ def plugin_card(context, result) -> str:
         pass
 
     if force_text_preview:
-        preview = _render_text_file_preview(getattr(result, "snapshot_dir", None), raw_output_path, plugin, icon_html)
+        preview = _render_text_file_preview(result.snapshot_dir, raw_output_path, plugin, icon_html)
         if preview:
             return mark_safe(preview)
 
@@ -608,7 +616,7 @@ def plugin_card(context, result) -> str:
 def output_card(snapshot, output_path: str, plugin: str) -> str:
     plugin_name = get_plugin_name(plugin)
     icon_html = get_plugin_icon(plugin_name)
-    preview = _render_text_file_preview(getattr(snapshot, "output_dir", None), output_path, plugin_name, icon_html)
+    preview = _render_text_file_preview(snapshot.output_dir, output_path, plugin_name, icon_html)
     if preview:
         return mark_safe(preview)
 
@@ -624,7 +632,9 @@ def plugin_full(context, result) -> str:
 
     Usage: {% plugin_full result %}
     """
-    if result is None or not hasattr(result, "plugin"):
+    from archivebox.core.models import ArchiveResult
+
+    if result is None or not isinstance(result, ArchiveResult):
         return ""
 
     plugin = get_plugin_name(result.plugin)
@@ -634,14 +644,13 @@ def plugin_full(context, result) -> str:
         return ""
 
     raw_output_path = ""
-    if hasattr(result, "embed_path_db"):
-        raw_output_path = result.embed_path_db() or ""
-    if not raw_output_path and hasattr(result, "embed_path"):
+    raw_output_path = result.embed_path_db() or ""
+    if not raw_output_path:
         raw_output_path = result.embed_path() or ""
     if _is_root_snapshot_output_path(raw_output_path):
         return ""
     output_url = build_snapshot_url(
-        str(getattr(result, "snapshot_id", "")),
+        str(result.snapshot_id),
         raw_output_path,
         request=context.get("request"),
         config=context.get("CONFIG"),
@@ -685,7 +694,7 @@ def api_token(context) -> str:
     from archivebox.api.auth import get_or_create_api_token
 
     request = context.get("request")
-    user = getattr(request, "user", None)
+    user = request.user
     if not user or not user.is_authenticated:
         return ""
 

archivebox/core/views.py

@@ -5,9 +5,7 @@ import os
 import posixpath
 from glob import glob, escape
 from django.utils import timezone
-import inspect
 from typing import cast
-from collections.abc import Callable
 from pathlib import Path
 from urllib.parse import quote, urlparse
 
@@ -30,7 +28,17 @@ from admin_data_views.utils import render_with_table_view, render_with_item_view
 from abx_plugins.plugins.archivewebpage import replay_preview as archivewebpage_replay
 
 from archivebox.config import CONSTANTS, CONSTANTS_CONFIG, VERSION
-from archivebox.config.common import SENSITIVE_CONFIG_VALUE_REDACTED, get_config, get_all_configs, redact_sensitive_config
+from archivebox.config.common import (
+    SENSITIVE_CONFIG_VALUE_REDACTED,
+    find_config_default,
+    find_config_section,
+    find_config_source,
+    find_config_type,
+    get_config,
+    get_all_configs,
+    get_request_config,
+    redact_sensitive_config,
+)
 from archivebox.config.configset import BaseConfigSet
 from archivebox.misc.paginators import CountlessPaginator
 from archivebox.misc.util import (
@@ -80,16 +88,6 @@ from archivebox.plugins.views import get_config_definition_link
 from archivebox.progressmonitor.views import live_progress_view, progress_endpoint
 
 
-def _get_request_config(request: HttpRequest, *, resolve_plugins: bool = False):
-    request_config = getattr(request, "archivebox_config", None)
-    request_config_resolves_plugins = bool(getattr(request, "_archivebox_config_resolves_plugins", False))
-    if request_config is None or (resolve_plugins and not request_config_resolves_plugins):
-        request_config = get_config(resolve_plugins=resolve_plugins)
-        request.archivebox_config = request_config
-        request._archivebox_config_resolves_plugins = resolve_plugins
-    return request_config
-
-
 def _files_index_target(snapshot: Snapshot, archivefile: str | None) -> str:
     target = archivefile or ""
     if target == "index.html":
@@ -122,14 +120,14 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
 
 
 def _admin_login_redirect_or_forbidden(request: HttpRequest):
-    if _get_request_config(request).CONTROL_PLANE_ENABLED:
+    if get_request_config(request).CONTROL_PLANE_ENABLED:
         return redirect(f"/admin/login/?next={request.path}")
     return HttpResponseForbidden("ArchiveBox is running with the control plane disabled in this security mode.")
 
 
 class HomepageView(View):
     def get(self, request):
-        request_config = _get_request_config(request)
+        request_config = get_request_config(request)
         if request.user.is_authenticated and request_config.CONTROL_PLANE_ENABLED:
             return redirect("/admin/core/snapshot/")
 
@@ -200,7 +198,7 @@ class SnapshotView(View):
 
         # Reuse the middleware-attached config; never re-bootstrap from env + plugin
         # schemas just to render a snapshot page (that pays ~30ms for no reason).
-        runtime_config = _get_request_config(request)
+        runtime_config = get_request_config(request)
         snapshot._runtime_config = runtime_config
         snapshot_permissions = get_snapshot_permissions(snapshot)
         hidden_card_plugins = {"archivedotorg", "favicon", "title"}
@@ -734,7 +732,7 @@ def _replay_path_visible(request: HttpRequest, path: Path) -> bool:
     snapshot = Snapshot.objects.filter(id=snapshot_id).select_related("crawl", "crawl__created_by").first()
     if not snapshot or not can_view_snapshot(request, snapshot):
         return False
-    request.archivebox_config = _get_request_config(request, resolve_plugins=False)
+    request.archivebox_config = get_request_config(request, resolve_plugins=False)
     return True
 
 
@@ -845,11 +843,14 @@ def _serve_responses_path(request, responses_root: Path, rel_path: str, show_ind
 
 
 def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str = ""):
-    request_config = _get_request_config(request, resolve_plugins=False)
+    rel_path = path or ""
+    request_config = get_request_config(
+        request,
+        resolve_plugins=rel_path.startswith("replay/") or rel_path == "replay",
+    )
     request.archivebox_config = request_config
     request.archivebox_snapshot_url = snapshot.url
     snapshot._runtime_config = request_config
-    rel_path = path or ""
 
     if rel_path.startswith("replay/") or rel_path == "replay":
         response = archivewebpage_replay.serve_replay_asset_response(rel_path, request_config, HttpResponse)
@@ -899,7 +900,8 @@ def _serve_snapshot_replay(request: HttpRequest, snapshot: Snapshot, path: str =
 
 
 def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str = ""):
-    request_config = _get_request_config(request)
+    request_config = get_request_config(request, resolve_plugins=False)
+    request.archivebox_config = request_config
     requested_root_index = path in ("", "index.html") or path.endswith("/")
     rel_path = path or ""
     if not rel_path or rel_path.endswith("/"):
@@ -909,13 +911,13 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
         raise Http404
 
     domain = domain.lower()
-    match = _latest_response_match(request, domain, rel_path, data_root=request_config.USERS_DIR)
+    match = _latest_response_match(request, domain, rel_path, data_root=CONSTANTS.USERS_DIR)
     if not match and "." not in Path(rel_path).name:
         index_path = f"{rel_path.rstrip('/')}/index.html"
-        match = _latest_response_match(request, domain, index_path, data_root=request_config.USERS_DIR)
+        match = _latest_response_match(request, domain, index_path, data_root=CONSTANTS.USERS_DIR)
     if not match and "." not in Path(rel_path).name:
         html_path = f"{rel_path}.html"
-        match = _latest_response_match(request, domain, html_path, data_root=request_config.USERS_DIR)
+        match = _latest_response_match(request, domain, html_path, data_root=CONSTANTS.USERS_DIR)
 
     show_indexes = bool(request.GET.get("files"))
     if match:
@@ -924,7 +926,7 @@ def _serve_original_domain_replay(request: HttpRequest, domain: str, path: str =
         if response is not None:
             return response
 
-    responses_root = _latest_responses_root(request, domain, data_root=request_config.USERS_DIR)
+    responses_root = _latest_responses_root(request, domain, data_root=CONSTANTS.USERS_DIR)
     if responses_root:
         response = _serve_responses_path(request, responses_root, rel_path, show_indexes)
         if response is not None:
@@ -946,7 +948,7 @@ class SnapshotHostView(View):
     """Serve snapshot directory contents on <snapshot-subdomain>.<listen_host>/<path>."""
 
     def get(self, request, snapshot_id: str, path: str = ""):
-        request_config = _get_request_config(request)
+        request_config = get_request_config(request)
         snapshot = _find_snapshot_by_ref(snapshot_id)
 
         if not snapshot:
@@ -998,15 +1000,15 @@ class PublicIndexView(ListView):
     paginator_class = CountlessPaginator
 
     def get_paginate_by(self, queryset):
-        runtime_config = getattr(self, "runtime_config", None)
+        runtime_config = self.__dict__.get("runtime_config")
         if runtime_config is None:
-            self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
+            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
         return runtime_config.SNAPSHOTS_PER_PAGE
 
     def get_context_data(self, **kwargs):
-        runtime_config = getattr(self, "runtime_config", None)
+        runtime_config = self.__dict__.get("runtime_config")
         if runtime_config is None:
-            self.runtime_config = runtime_config = _get_request_config(self.request, resolve_plugins=False)
+            self.runtime_config = runtime_config = get_request_config(self.request, resolve_plugins=False)
         search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
         search_mode_backend = get_search_mode_backend(search_mode, config=runtime_config)
         context = {
@@ -1023,7 +1025,7 @@ class PublicIndexView(ListView):
             self.request.GET.get("q")
             and get_search_mode_base(search_mode, config=runtime_config) == "deep"
             and search_mode_backend
-            and getattr(context.get("paginator"), "count", 0) == 0,
+            and context["paginator"].count == 0,
         )
         snapshots = list(context.get("object_list") or ())
         icons_by_snapshot: dict[str, set[str]] = {str(snapshot.id): set() for snapshot in snapshots}
@@ -1086,7 +1088,7 @@ class PublicIndexView(ListView):
         if not query:
             return qs
 
-        runtime_config = getattr(self, "runtime_config", None)
+        runtime_config = self.__dict__.get("runtime_config")
         search_mode = get_search_mode(self.request.GET.get("search_mode"), config=runtime_config)
         try:
             return apply_snapshot_search(
@@ -1105,7 +1107,7 @@ class PublicIndexView(ListView):
     def get(self, *args, **kwargs):
         if self.request.user.is_authenticated:
             return redirect("/admin/core/snapshot/")
-        if _get_request_config(self.request).PUBLIC_INDEX:
+        if get_request_config(self.request).PUBLIC_INDEX:
             response = super().get(*args, **kwargs)
             return response
         else:
@@ -1132,7 +1134,7 @@ class AddView(UserPassesTestMixin, FormView):
         return kwargs
 
     def test_func(self):
-        return _get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
+        return get_request_config(self.request).PUBLIC_ADD_VIEW or self.request.user.is_authenticated
 
     def _can_override_crawl_config(self) -> bool:
         return is_admin_user(self.request)
@@ -1146,11 +1148,11 @@ class AddView(UserPassesTestMixin, FormView):
         if not self._can_override_crawl_config():
             return {}
 
-        return custom_config
+        return {str(key): value for key, value in custom_config.items() if not str(key).endswith("_BINARY")}
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-        request_config = _get_request_config(self.request, resolve_plugins=True)
+        request_config = get_request_config(self.request, resolve_plugins=True)
         required_search_plugin = f"search_backend_{request_config.SEARCH_BACKEND_ENGINE}".strip()
         can_override_crawl_config = self._can_override_crawl_config()
         plugin_configs = discover_plugin_configs() if can_override_crawl_config else {}
@@ -1252,7 +1254,7 @@ class AddView(UserPassesTestMixin, FormView):
 
                 created_by_id = get_or_create_system_user_pk()
 
-        created_by_name = getattr(self.request.user, "username", "web") if self.request.user.is_authenticated else "web"
+        created_by_name = self.request.user.username if self.request.user.is_authenticated else "web"
 
         # 1. save the provided urls to sources/2024-11-05__23-59-59__web_ui_add_by_user_<user_pk>.txt
         sources_file = CONSTANTS.SOURCES_DIR / f"{timezone.now().strftime('%Y-%m-%d__%H-%M-%S')}__web_ui_add_by_user_{created_by_id}.txt"
@@ -1267,8 +1269,7 @@ class AddView(UserPassesTestMixin, FormView):
         config = {}
         if plugins:
             config["PLUGINS"] = plugins
-        request_user = self.request.user if self.request.user.is_authenticated else None
-        effective_config = get_config(persona=persona, user=request_user) if persona else get_config(user=request_user)
+        effective_config = get_config(persona=persona) if persona else get_config()
         if crawl_max_concurrent_snapshots != int(effective_config.CRAWL_MAX_CONCURRENT_SNAPSHOTS):
             config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] = crawl_max_concurrent_snapshots
         if delete_after != str(effective_config.DELETE_AFTER):
@@ -1378,11 +1379,7 @@ class WebAddView(AddView):
                 return redirect(f"/{snapshot.url_path}")
 
         request_host = (request.get_host() or "").lower()
-        if (
-            request.user.is_authenticated
-            and not _get_request_config(request).PUBLIC_ADD_VIEW
-            and host_matches(request_host, get_web_host())
-        ):
+        if request.user.is_authenticated and not get_request_config(request).PUBLIC_ADD_VIEW and host_matches(request_host, get_web_host()):
             return redirect(build_admin_url(request.get_full_path(), request=request))
 
         if not self.test_func():
@@ -1461,79 +1458,11 @@ class HealthCheckView(View):
         return HttpResponse("OK", content_type="text/plain", status=200)
 
 
-def find_config_section(key: str) -> str:
-    CONFIGS = get_all_configs()
-
-    if key in CONSTANTS_CONFIG:
-        return "CONSTANT"
-    matching_sections = [section_id for section_id, section in CONFIGS.items() if key in dict(section)]
-    section = matching_sections[0] if matching_sections else "DYNAMIC"
-    return section
-
-
-def find_config_default(key: str) -> str:
-    CONFIGS = get_all_configs()
-
-    if key in CONSTANTS_CONFIG:
-        return str(CONSTANTS_CONFIG[key])
-
-    default_val = None
-
-    for config in CONFIGS.values():
-        if key in dict(config):
-            default_val = type(config).model_fields[key].default
-            break
-
-    if isinstance(default_val, Callable):
-        default_val = inspect.getsource(default_val).split("lambda", 1)[-1].split(":", 1)[-1].replace("\n", " ").strip()
-        if default_val.count(")") > default_val.count("("):
-            default_val = default_val[:-1]
-    else:
-        default_val = str(default_val)
-
-    return default_val
-
-
-def find_config_type(key: str) -> str:
-    CONFIGS = get_all_configs()
-
-    for config in CONFIGS.values():
-        if key in type(config).model_fields:
-            annotation = type(config).model_fields[key].annotation
-            return getattr(annotation, "__name__", str(annotation))
-    return "str"
-
-
-def find_config_source(key: str, merged_config: dict) -> str:
-    """Determine where a config value comes from."""
-    from archivebox.machine.models import Machine
-
-    # Environment variables override all persistent config sources.
-    if key in os.environ:
-        return "Environment"
-
-    # Machine.config overrides ArchiveBox.conf.
-    try:
-        machine = Machine.current()
-        if machine.config and key in machine.config:
-            return "Machine"
-    except Exception:
-        pass
-
-    # Check if it's from archivebox.config.file
-    file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
-    if key in file_config:
-        return "Config File"
-
-    # Otherwise it's using the default
-    return "Default"
-
-
 @render_with_table_view
 def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
     CONFIGS = get_all_configs()
 
-    assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
 
     merged_config = get_config(redact_sensitive=True)
 
@@ -1555,12 +1484,12 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
             rows["Type"].append(format_html("<code>{}</code>", find_config_type(key)))
 
             # Use merged config value (includes machine overrides)
-            actual_value = merged_config.get(key, getattr(section, key, None))
+            actual_value = merged_config.get(key, dict(section)[key])
             rows["Value"].append(mark_safe(f"<code>{actual_value}</code>"))
 
             # Show where the value comes from
             source = find_config_source(key, merged_config)
-            source_colors = {"Machine": "purple", "Environment": "blue", "Config File": "green", "Default": "gray"}
+            source_colors = {"Machine": "purple", "Environment": "blue", "File": "green", "Plugin Default": "teal", "Default": "gray"}
             rows["Source"].append(format_html('<code style="color: {}">{}</code>', source_colors.get(source, "gray"), source))
 
             rows["Default"].append(
@@ -1575,7 +1504,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
     for key in CONSTANTS_CONFIG.keys():
         rows["Section"].append(section)  # section.replace('_', ' ').title().replace(' Config', '')
         rows["Key"].append(ItemLink(key, key=key))
-        rows["Type"].append(format_html("<code>{}</code>", getattr(type(CONSTANTS_CONFIG[key]), "__name__", str(CONSTANTS_CONFIG[key]))))
+        rows["Type"].append(format_html("<code>{}</code>", type(CONSTANTS_CONFIG[key]).__name__))
         rows["Value"].append(format_html("<code>{}</code>", redact_sensitive_config(CONSTANTS_CONFIG).get(key)))
         rows["Source"].append(mark_safe('<code style="color: gray">Constant</code>'))
         rows["Default"].append(
@@ -1598,17 +1527,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
 
     CONFIGS = get_all_configs()
 
-    assert getattr(request.user, "is_superuser", False), "Must be a superuser to view configuration settings."
+    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
 
     merged_config = get_config(redact_sensitive=True)
 
     # Determine all sources for this config value
     sources_info = []
 
-    # Environment variable
-    if key in os.environ:
-        sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
-
     # Machine config
     machine = None
     machine_admin_url = None
@@ -1620,11 +1545,15 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
     except Exception:
         pass
 
+    # Environment variable
+    if key in os.environ:
+        sources_info.append(("Environment", redact_sensitive_config(os.environ).get(key), "blue"))
+
     # Config file value
     if CONSTANTS.CONFIG_FILE.exists():
         file_config = BaseConfigSet.load_from_file(CONSTANTS.CONFIG_FILE)
         if key in file_config:
-            sources_info.append(("Config File", redact_sensitive_config(file_config).get(key), "green"))
+            sources_info.append(("File", redact_sensitive_config(file_config).get(key), "green"))
 
     # Default value
     default_val = find_config_default(key)

archivebox/core/widgets.py

@@ -4,6 +4,8 @@ import json
 import re
 import hashlib
 from django import forms
+from django.db.models.manager import BaseManager
+from django.db.models.query import QuerySet
 from django.utils.html import escape
 from django.utils.safestring import mark_safe
 
@@ -65,10 +67,12 @@ class TagEditorWidget(forms.Widget):
         # Parse value to get list of tag names
         tags = []
         if value:
-            if hasattr(value, "all"):  # QuerySet
+            if isinstance(value, (BaseManager, QuerySet)):
                 tags = sorted([tag.name for tag in value.all()])
             elif isinstance(value, (list, tuple)):
-                if value and hasattr(value[0], "name"):  # List of Tag objects
+                from archivebox.core.models import Tag
+
+                if value and isinstance(value[0], Tag):  # List of Tag objects
                     tags = sorted([tag.name for tag in value])
                 else:  # List of strings or IDs
                     # Could be tag IDs from form submission
@@ -675,12 +679,14 @@ class InlineTagEditorWidget(TagEditorWidget):
         # Parse value to get list of tag dicts with id and name
         tag_data = []
         if value:
-            if hasattr(value, "all"):  # QuerySet
+            if isinstance(value, (BaseManager, QuerySet)):
                 for tag in value.all():
                     tag_data.append({"id": tag.pk, "name": tag.name})
                 tag_data.sort(key=lambda x: x["name"].lower())
             elif isinstance(value, (list, tuple)):
-                if value and hasattr(value[0], "name"):
+                from archivebox.core.models import Tag
+
+                if value and isinstance(value[0], Tag):
                     for tag in value:
                         tag_data.append({"id": tag.pk, "name": tag.name})
                     tag_data.sort(key=lambda x: x["name"].lower())

archivebox/crawls/admin.py

@@ -605,9 +605,7 @@ class CrawlAdminForm(forms.ModelForm):
         if commit:
             instance.save()
             instance.apply_crawl_config_filters()
-            save_m2m = getattr(self, "_save_m2m", None)
-            if callable(save_m2m):
-                save_m2m()
+            self._save_m2m()
         return instance
 
 
@@ -729,7 +727,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         super().__init__(model, admin_site)
         self.crawl_admin_base_config = None
         self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
 
     class Media:
         css = {"all": ("admin/crawls/crawl_change.css",)}
@@ -739,9 +736,8 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         self.request = request
         self.crawl_admin_base_config = request.archivebox_config
         self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
         response = super().changelist_view(request, extra_context)
-        cl = getattr(response, "context_data", {}).get("cl") if hasattr(response, "context_data") else None
+        cl = response.context_data.get("cl")
         if cl is not None and not self.should_annotate_snapshot_counts(request):
             self.hydrate_visible_snapshot_counts(cl.result_list)
         return response
@@ -790,7 +786,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         self.request = request
         self.crawl_admin_base_config = request.archivebox_config
         self.stop_reason_cache = {}
-        self.persona_limit_config_cache = {}
         crawl = self.get_object(request, object_id)
         if crawl:
             self.hydrate_visible_snapshot_counts([crawl])
@@ -982,7 +977,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         if obj.pk in self.stop_reason_cache:
             return self.stop_reason_cache[obj.pk]
 
-        output_dir = obj.output_dir_for_config(self.crawl_admin_base_config)
+        output_dir = obj.output_dir
         config = self.limit_config_for_crawl(obj, output_dir)
         reason = obj.stop_reason(
             config=config,
@@ -994,22 +989,13 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
         return reason
 
     def limit_config_for_crawl(self, obj, output_dir):
-        config = {
-            "CRAWL_DIR": str(output_dir),
-            "CRAWL_MAX_URLS": self.crawl_admin_base_config.CRAWL_MAX_URLS,
-            "CRAWL_MAX_SIZE": self.crawl_admin_base_config.CRAWL_MAX_SIZE,
-            "CRAWL_TIMEOUT": self.crawl_admin_base_config.CRAWL_TIMEOUT,
-            "SNAPSHOT_MAX_SIZE": self.crawl_admin_base_config.SNAPSHOT_MAX_SIZE,
-        }
-        if obj.persona_id:
-            if obj.persona_id not in self.persona_limit_config_cache:
-                self.persona_limit_config_cache[obj.persona_id] = {
-                    key: value for key, value in obj.persona.get_derived_config().items() if key in config
-                }
-            config.update(self.persona_limit_config_cache[obj.persona_id])
-        if obj.config:
-            config.update({key: value for key, value in obj.config.items() if key in config})
-        return config
+        from archivebox.config.common import get_config
+
+        return get_config(crawl=obj).for_crawl_runtime(
+            crawl=obj,
+            persona=obj.resolve_persona(),
+            crawl_output_dir=output_dir,
+        )
 
     @admin.display(description="Status", ordering="status")
     def status_with_stop_reason(self, obj):
@@ -1106,12 +1092,10 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
 
     @admin.display(description="Snapshots")
     def snapshots_changelist(self, obj):
-        request = getattr(self, "request", None)
+        request = self.request
         snapshot_changelist = reverse("admin:core_snapshot_changelist")
         scoped_params = {"crawl_id": str(obj.pk)}
         full_url = f"{snapshot_changelist}?{urlencode(scoped_params)}"
-        if request is None:
-            return format_html('<a class="button" href="{}">Open snapshots changelist</a>', full_url)
 
         snapshot_admin = self.admin_site._registry[Snapshot]
         changelist_request = copy(request)
@@ -1305,7 +1289,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
         return self.fieldsets
 
     def save_model(self, request, obj, form, change):
-        if not obj.created_by_id and getattr(request, "user", None) and request.user.is_authenticated:
+        if not obj.created_by_id and request.user.is_authenticated:
             obj.created_by = request.user
         super().save_model(request, obj, form, change)
 
@@ -1315,14 +1299,14 @@ class CrawlScheduleAdmin(BaseModelAdmin):
 
     @admin.display(description="# Crawls", ordering="crawl_count")
     def num_crawls(self, obj):
-        count = getattr(obj, "crawl_count", None)
+        count = obj.__dict__.get("crawl_count")
         if count is None:
             count = obj.crawl_set.count()
         return count
 
     @admin.display(description="# Snapshots", ordering="snapshot_count")
     def num_snapshots(self, obj):
-        count = getattr(obj, "snapshot_count", None)
+        count = obj.__dict__.get("snapshot_count")
         if count is None:
             count = Snapshot.objects.filter(crawl__schedule=obj).count()
         return count
@@ -1338,7 +1322,7 @@ class CrawlScheduleAdmin(BaseModelAdmin):
         crawl_ids = obj.crawl_set.values_list("pk", flat=True)
         return render_snapshots_list(
             Snapshot.objects.filter(crawl_id__in=crawl_ids),
-            request=getattr(self, "request", None),
+            request=self.request,
             prefix="schedule_snapshots",
         )
 

archivebox/crawls/migrations/0016_hydrate_crawl_permissions.py

@@ -1,5 +1,6 @@
 import os
 import json
+import uuid
 
 from django.db import migrations
 from django.db.models import Q
@@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
 BATCH_SIZE = 1000
 
 
-def legacy_bool(value):
-    if value is None:
-        return None
-    normalized = str(value).strip().lower()
-    if normalized in {"1", "true", "yes", "on"}:
-        return True
-    if normalized in {"0", "false", "no", "off"}:
-        return False
-    return None
-
-
-def permissions_from_legacy_public_flags(config):
-    if str(config.get("PERMISSIONS") or "").strip():
-        return None
-    public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
-    public_index = legacy_bool(config.get("PUBLIC_INDEX"))
-    if public_snapshots is False:
-        return "private"
-    if public_index is False:
-        return "unlisted"
-    if public_snapshots is True or public_index is True:
-        return "public"
-    return None
-
-
 def normalize_permissions(value, default):
     value = str(value or "").strip().lower()
     return value if value in VALID_PERMISSIONS else default
@@ -59,22 +35,19 @@ def raw_base_config(apps):
 
 
 def resolve_permissions(config, default):
+    from archivebox.config.common import permissions_from_legacy_public_flags
+
     explicit = str(config.get("PERMISSIONS") or "").strip().lower()
     if explicit in VALID_PERMISSIONS:
         return explicit
     return permissions_from_legacy_public_flags(config) or default
 
 
-def model_has_config(model):
-    try:
-        model._meta.get_field("config")
-    except Exception:
-        return False
-    return True
-
-
 def id_values(pk):
-    return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
+    if isinstance(pk, uuid.UUID):
+        return str(pk), pk.hex
+    pk_str = str(pk)
+    return pk_str, pk_str.replace("-", "")
 
 
 def flush_batch(cursor, table_name, batch):
@@ -125,8 +98,6 @@ def _ensure_permissions_column(cursor):
 
 def hydrate_crawl_permissions(apps, schema_editor):
     Crawl = apps.get_model("crawls", "Crawl")
-    User = apps.get_model("auth", "User")
-    user_has_config = model_has_config(User)
     base_config = raw_base_config(apps)
     default_permissions = resolve_permissions(base_config, "public")
     table_name = schema_editor.quote_name(Crawl._meta.db_table)
@@ -135,17 +106,13 @@ def hydrate_crawl_permissions(apps, schema_editor):
     batch = []
     missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
 
-    for crawl in Crawl.objects.filter(missing_permissions).select_related("persona", "created_by").iterator(chunk_size=BATCH_SIZE):
+    for crawl in Crawl.objects.filter(missing_permissions).select_related("persona").iterator(chunk_size=BATCH_SIZE):
         config = dict(crawl.config or {})
         resolved = dict(base_config)
         if crawl.persona_id:
             persona_config = crawl.persona.config or {}
             if isinstance(persona_config, dict):
                 resolved.update(persona_config)
-        if user_has_config:
-            user_config = crawl.created_by.config or {}
-            if isinstance(user_config, dict):
-                resolved.update(user_config)
         resolved.update(config)
         config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
         batch.append((crawl.id, config))

archivebox/crawls/migrations/0018_freeze_crawl_config_snapshots.py

@@ -1,6 +1,5 @@
 import json
 
-from django.conf import settings
 from django.db import migrations
 
 
@@ -19,26 +18,31 @@ def _flush_updates(Crawl, db_alias, pending):
 
 def freeze_existing_crawl_configs(apps, schema_editor):
     from archivebox.config.common import build_crawl_config_snapshot
-    from archivebox.personas.models import Persona
+    from archivebox.config.constants import CONSTANTS
+    from archivebox.personas.models import derive_persona_config
+
+    class PersonaConfigSnapshot:
+        def __init__(self, persona):
+            self.name = persona.name
+            self.config = dict(persona.config or {})
+
+        def get_derived_config(self):
+            return derive_persona_config(name=self.name, config=self.config, persona_dir=CONSTANTS.PERSONAS_DIR / self.name)
 
     Crawl = apps.get_model("crawls", "Crawl")
-    auth_app_label, auth_model_name = settings.AUTH_USER_MODEL.split(".", 1)
-    User = apps.get_model(auth_app_label, auth_model_name)
+    Persona = apps.get_model("personas", "Persona")
     db_alias = schema_editor.connection.alias
-    rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "created_by_id", "config")
-    persona_ids = {persona_id for _, persona_id, _, _ in rows if persona_id}
-    user_ids = {user_id for _, _, user_id, _ in rows if user_id}
-    personas = {persona.pk: persona for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
-    users = {user.pk: user for user in User.objects.using(db_alias).filter(pk__in=user_ids)}
+    rows = Crawl.objects.using(db_alias).values_list("id", "persona_id", "config")
+    persona_ids = {persona_id for _, persona_id, _ in rows if persona_id}
+    personas = {persona.pk: PersonaConfigSnapshot(persona) for persona in Persona.objects.using(db_alias).filter(pk__in=persona_ids)}
 
     frozen_cache = {}
     pending = []
-    for crawl_id, persona_id, user_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
+    for crawl_id, persona_id, current_config in rows.iterator(chunk_size=BATCH_SIZE):
         current_config = dict(current_config or {})
-        cache_key = (persona_id, user_id, _config_cache_key(current_config))
+        cache_key = (persona_id, _config_cache_key(current_config))
         if cache_key not in frozen_cache:
             frozen_cache[cache_key] = build_crawl_config_snapshot(
-                user=users.get(user_id),
                 persona=personas.get(persona_id),
                 overrides=current_config,
             )

archivebox/crawls/models.py

@@ -109,11 +109,10 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
         template = self.template
         label = template.label or self.label
         persona = template.persona if template.persona_id else None
-        user = template.created_by if template.created_by_id else None
 
         return Crawl.objects.create(
             urls=template.urls,
-            config=build_crawl_config_snapshot(user=user, persona=persona, overrides=self.config or {}),
+            config=build_crawl_config_snapshot(persona=persona, overrides=self.config or {}),
             max_depth=template.max_depth,
             tags_str=template.tags_str,
             persona_id=template.persona_id,
@@ -171,10 +170,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     INACTIVE_STATES = (StatusChoices.PAUSED, StatusChoices.SEALED)
 
     schedule_id: uuid.UUID | None
-    sm: "CrawlMachine"
 
     snapshot_set: models.Manager["Snapshot"]
 
+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> "CrawlMachine": ...
+
     class Meta(
         ModelWithDeleteAfter.Meta,
         ModelWithOutputDir.Meta,
@@ -197,9 +200,9 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         return f"[...{short_id}] {first_url[:120]}"
 
     def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
+        from archivebox.config.common import resolve_delete_after_config_value
 
-        return get_config(crawl=self).DELETE_AFTER
+        return resolve_delete_after_config_value(self.config)
 
     def pause(self, *, save: bool = True) -> bool:
         return super().pause(save=save)
@@ -286,15 +289,14 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         config = dict(self.config or {})
         is_new = self._state.adding or old_crawl is None
         persona = self.persona if self.persona_id else None
-        user = self.created_by if self.created_by_id else None
         if is_new:
             from archivebox.config.common import build_crawl_config_snapshot
 
-            config = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
+            config = build_crawl_config_snapshot(persona=persona, overrides=config)
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
 
-            config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, user=user, include_machine=True).PERMISSIONS)
+            config["PERMISSIONS"] = normalize_permissions(get_config(persona=persona, include_machine=True).PERMISSIONS)
         if "CRAWL_MAX_CONCURRENT_SNAPSHOTS" in config:
             raw_concurrency = config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"]
             if raw_concurrency in (None, ""):
@@ -469,11 +471,8 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         )
         return crawl
 
-    def output_dir_for_config(self, runtime_config: Mapping[str, Any] | Any) -> Path:
-        """
-        Construct output directory: archive/users/{username}/crawls/{YYYYMMDD}/{domain}/{crawl-id}
-        Domain is extracted from the first URL in the crawl.
-        """
+    @property
+    def output_dir(self) -> Path:
         from archivebox.config import CONSTANTS
         from archivebox.core.models import Snapshot
 
@@ -486,14 +485,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
                 break
         domain = Snapshot.extract_domain_from_url(first_url) if first_url else "unknown"
 
-        users_dir = Path(runtime_config["USERS_DIR"]) if isinstance(runtime_config, Mapping) else runtime_config.USERS_DIR
-        return users_dir / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
-
-    @property
-    def output_dir(self) -> Path:
-        from archivebox.config.common import get_config
-
-        output_dir = self.output_dir_for_config(get_config(resolve_plugins=False))
+        output_dir = CONSTANTS.USERS_DIR / self.created_by.username / CONSTANTS.CRAWLS_DIR_NAME / date_str / domain / str(self.id)
         hyphen_dir = output_dir.with_name(str(uuid.UUID(hex=self.id.hex)))
         return output_dir if output_dir.exists() or not hyphen_dir.exists() else hyphen_dir
 
@@ -783,7 +775,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     def _config_value(config: Mapping[str, Any] | Any, key: str, default: Any = None) -> Any:
         if isinstance(config, Mapping):
             return config.get(key, default)
-        return getattr(config, key, default)
+        return config[key] if key in config else default
 
     @classmethod
     def create_scheduler_row(cls, **kwargs) -> "Crawl":
@@ -794,17 +786,12 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         kwargs.setdefault("created_at", now)
         kwargs.setdefault("modified_at", now)
         config = normalize_config_json_values(kwargs.get("config") or {})
-        user = kwargs.get("created_by")
         persona = kwargs.get("persona")
-        if user is None and kwargs.get("created_by_id"):
-            from django.contrib.auth import get_user_model
-
-            user = get_user_model().objects.filter(pk=kwargs["created_by_id"]).first()
         if persona is None and kwargs.get("persona_id"):
             from archivebox.personas.models import Persona
 
             persona = Persona.objects.filter(pk=kwargs["persona_id"]).first()
-        kwargs["config"] = build_crawl_config_snapshot(user=user, persona=persona, overrides=config)
+        kwargs["config"] = build_crawl_config_snapshot(persona=persona, overrides=config)
         crawl = cls(**kwargs)
         if crawl.delete_at is None:
             crawl.set_delete_at_from_config()
@@ -820,18 +807,20 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
     ) -> str:
         from abx_dl.limits import CrawlLimitState
 
+        if output_dir is None:
+            output_dir = self.output_dir
         if config is None:
             from archivebox.config.common import get_config
 
-            config = get_config(crawl=self, include_machine=False)
-        if output_dir is None:
-            output_dir = self.output_dir
+            config = get_config(crawl=self, include_machine=False).for_crawl_runtime(
+                crawl=self,
+                persona=self.resolve_persona(),
+                crawl_output_dir=output_dir,
+            )
 
         limits_path = output_dir / ".abx-dl" / "limits.json"
         if limits_path.exists():
-            config_with_crawl_dir = {**dict(config.items())} if isinstance(config, Mapping) else config
-            config_with_crawl_dir["CRAWL_DIR"] = str(output_dir)
-            stop_reason = CrawlLimitState.from_config(config_with_crawl_dir).get_stop_reason()
+            stop_reason = CrawlLimitState.from_config(config).get_stop_reason()
             if stop_reason:
                 return stop_reason
 
@@ -1302,10 +1291,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
         from archivebox.machine.models import Binary, Machine
 
         def get_runtime_config():
-            config = get_config(crawl=self)
-            if persona_runtime_overrides:
-                config.update(persona_runtime_overrides)
-            return config
+            return get_config(crawl=self).for_crawl_runtime(
+                crawl=self,
+                persona=persona,
+                runtime_overrides=persona_runtime_overrides,
+            )
 
         system_task = self.get_system_task()
         if system_task == "archivebox://update":

archivebox/machine/admin.py

@@ -590,9 +590,10 @@ class ProcessAdmin(BaseModelAdmin):
 
     @admin.display(description="ArchiveResult", ordering="archiveresult__plugin")
     def archiveresult_link(self, process):
-        if not hasattr(process, "archiveresult"):
+        try:
+            ar = process.archiveresult
+        except Process.archiveresult.RelatedObjectDoesNotExist:
             return "-"
-        ar = process.archiveresult
         return format_html(
             '<a href="/admin/core/archiveresult/{}/change">{} ← <code>{}</code></a>',
             ar.id,
@@ -602,9 +603,9 @@ class ProcessAdmin(BaseModelAdmin):
 
     @admin.display(description="Snapshot", ordering="archiveresult__snapshot__id")
     def snapshot_link(self, process):
-        ar = getattr(process, "archiveresult", None)
-        snapshot = getattr(ar, "snapshot", None)
-        if not snapshot:
+        try:
+            snapshot = process.archiveresult.snapshot
+        except Process.archiveresult.RelatedObjectDoesNotExist:
             return "-"
         return format_html(
             '<a href="/admin/core/snapshot/{}/change"><code>{}</code></a>',
@@ -614,10 +615,9 @@ class ProcessAdmin(BaseModelAdmin):
 
     @admin.display(description="Crawl", ordering="archiveresult__snapshot__crawl__id")
     def crawl_link(self, process):
-        ar = getattr(process, "archiveresult", None)
-        snapshot = getattr(ar, "snapshot", None)
-        crawl = getattr(snapshot, "crawl", None)
-        if not crawl:
+        try:
+            crawl = process.archiveresult.snapshot.crawl
+        except Process.archiveresult.RelatedObjectDoesNotExist:
             return "-"
         return format_html(
             '<a href="/admin/crawls/crawl/{}/change"><code>{}</code></a>',
@@ -682,7 +682,10 @@ class ProcessAdmin(BaseModelAdmin):
 
     @admin.display(description="Output", ordering="archiveresult__output_size")
     def output_summary(self, process):
-        output_files = getattr(getattr(process, "archiveresult", None), "output_files", {}) or {}
+        try:
+            output_files = process.archiveresult.output_files or {}
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            output_files = {}
 
         if isinstance(output_files, str):
             try:

archivebox/machine/models.py

@@ -19,6 +19,7 @@ from django.db.models import Q, QuerySet
 from django.utils import timezone
 from django.utils.functional import cached_property
 
+from archivebox.config import CONSTANTS
 from archivebox.config.common import rprint
 from archivebox.base_models.models import ModelWithDeleteAfter, ModelWithHealthStats, normalize_config_json_values
 from archivebox.workers.models import BaseStateMachine, ModelWithStateMachine
@@ -411,14 +412,25 @@ class NetworkInterface(ModelWithHealthStats):
     def current(cls, refresh: bool = False) -> NetworkInterface:
         global _CURRENT_INTERFACE
         machine = Machine.current(refresh=refresh)
-        if _CURRENT_INTERFACE:
-            if (
-                not refresh
-                and _CURRENT_INTERFACE.machine_id == machine.id
-                and timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL)
-            ):
+        if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
+            if not refresh:
+                # Callers that pass refresh=False are asking for attribution to
+                # the currently known interface, not for public-IP/ISP probing.
+                # Maintenance paths create many short-lived services per run;
+                # expiring this in-memory object by age forced every crawl to
+                # hit external network APIs even though Process rows only need
+                # a stable existing FK. Active downloading paths opt into live
+                # detection with refresh=True.
                 return _CURRENT_INTERFACE
-            _CURRENT_INTERFACE = None
+            if timezone.now() < _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
+                return _CURRENT_INTERFACE
+        _CURRENT_INTERFACE = None
+
+        if not refresh:
+            _CURRENT_INTERFACE = cls.objects.filter(machine=machine).order_by("-modified_at", "-created_at").first()
+            if _CURRENT_INTERFACE is not None:
+                return _CURRENT_INTERFACE
+
         net_info = get_host_network()
         lookup = dict(
             machine=machine,
@@ -442,7 +454,7 @@ class NetworkInterface(ModelWithHealthStats):
             if refresh or timezone.now() >= _CURRENT_INTERFACE.modified_at + timedelta(seconds=NETWORK_INTERFACE_RECHECK_INTERVAL):
                 updates = ["modified_at"]
                 for key, value in net_info.items():
-                    if getattr(_CURRENT_INTERFACE, key) != value:
+                    if _CURRENT_INTERFACE.__dict__.get(key) != value:
                         setattr(_CURRENT_INTERFACE, key, value)
                         updates.append(key)
                 if len(updates) > 1:
@@ -547,9 +559,15 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
 
     state_machine_name: str | None = "archivebox.machine.models.BinaryMachine"
     active_state: str = StatusChoices.QUEUED
+    warn_on_save_outside_runner = False
 
     objects = BinaryManager()  # pyright: ignore[reportIncompatibleVariableOverride]
 
+    if TYPE_CHECKING:
+
+        @property
+        def sm(self) -> BinaryMachine: ...
+
     class Meta(ModelWithHealthStats.Meta, ModelWithStateMachine.Meta):
         app_label = "machine"
         verbose_name = "Binary"
@@ -581,10 +599,7 @@ class Binary(ModelWithHealthStats, ModelWithStateMachine):
         Get output directory for this binary's hook logs.
         Path: data/machines/{machine_uuid}/binaries/{binary_name}/{binary_uuid}
         """
-        from archivebox.config.common import get_config
-
-        data_dir = get_config().DATA_DIR
-        return data_dir / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
+        return CONSTANTS.DATA_DIR / "machines" / str(self.machine_id) / "binaries" / self.name / str(self.id)
 
     def to_json(self) -> dict:
         """
@@ -1144,15 +1159,13 @@ class Process(ModelWithDeleteAfter, models.Model):
         return f"Process[{self.id}] {cmd_str} ({self.status})"
 
     def get_delete_after_config_value(self):
-        from archivebox.config.common import get_config
-
         value = self.env.get("DELETE_AFTER")
         if value not in (None, ""):
             return value
         value = (self.machine.config or {}).get("DELETE_AFTER")
         if value not in (None, ""):
             return value
-        return get_config(include_machine=False).DELETE_AFTER
+        return "0"
 
     @classmethod
     def missing_delete_at_candidates(cls):
@@ -1174,17 +1187,18 @@ class Process(ModelWithDeleteAfter, models.Model):
     @property
     def plugin(self) -> str:
         """Get plugin name from associated ArchiveResult (if any)."""
-        if hasattr(self, "archiveresult"):
-            # Inline import to avoid circular dependency
+        try:
             return self.archiveresult.plugin
-        return ""
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return ""
 
     @property
     def hook_name(self) -> str:
         """Get hook name from associated ArchiveResult (if any)."""
-        if hasattr(self, "archiveresult"):
+        try:
             return self.archiveresult.hook_name
-        return ""
+        except Process.archiveresult.RelatedObjectDoesNotExist:
+            return ""
 
     def to_json(self) -> dict:
         """
@@ -2449,16 +2463,14 @@ class Process(ModelWithDeleteAfter, models.Model):
         """
         import subprocess
         from importlib.resources import files
-        from archivebox.config.common import get_config
 
         chrome_utils = files("abx_plugins.plugins.chrome").joinpath("chrome_utils.js")
         if not chrome_utils.exists():
             return 0
 
-        config = get_config()
         crawl_roots = [
             crawls_dir
-            for user_dir in config.USERS_DIR.iterdir()
+            for user_dir in CONSTANTS.USERS_DIR.iterdir()
             if user_dir.is_dir()
             for crawls_dir in [user_dir / "crawls"]
             if crawls_dir.is_dir()

archivebox/misc/checks.py

@@ -66,7 +66,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:
     from archivebox.config.paths import create_and_chown_dir, get_or_create_working_tmp_dir, get_or_create_working_lib_dir
 
     config = config or get_config(**config_kwargs)
-    archive_dir = config.ARCHIVE_DIR
+    archive_dir = CONSTANTS.ARCHIVE_DIR
     archive_dir_exists = os.path.isdir(archive_dir)
     if not archive_dir_exists:
         print("[red][X] No archivebox index found in the current directory.[/red]", file=sys.stderr)
@@ -82,7 +82,7 @@ def check_data_folder(config=None, **config_kwargs) -> None:
 
     # Create data dir subdirs
     create_and_chown_dir(CONSTANTS.SOURCES_DIR)
-    create_and_chown_dir(config.USERS_DIR)
+    create_and_chown_dir(CONSTANTS.USERS_DIR)
     create_and_chown_dir(CONSTANTS.PERSONAS_DIR / "Default")
     create_and_chown_dir(CONSTANTS.LOGS_DIR)
     # create_and_chown_dir(CONSTANTS.CACHE_DIR)
@@ -211,11 +211,10 @@ def check_not_inside_source_dir():
     """Prevent running ArchiveBox from inside its source directory (would pollute repo with data files)."""
     cwd = Path(os.getcwd()).resolve()
     is_source_dir = (cwd / "archivebox" / "__init__.py").exists() and (cwd / "pyproject.toml").exists()
-    data_dir_set_elsewhere = os.environ.get("DATA_DIR", "").strip() and Path(os.environ["DATA_DIR"]).resolve() != cwd
     is_testing = "pytest" in sys.modules or "unittest" in sys.modules
 
-    if is_source_dir and not data_dir_set_elsewhere and not is_testing:
-        raise SystemExit("[!] Cannot run from source dir, set DATA_DIR or cd to a data folder first")
+    if is_source_dir and not is_testing:
+        raise SystemExit("[!] Cannot run from source dir, cd to a data folder first")
 
 
 def check_data_dir_permissions(config=None, **config_kwargs):

archivebox/misc/db.py

@@ -1,7 +1,7 @@
 """
 Database utility functions for ArchiveBox.
 
-Post-bootstrap: requires archivebox.config (DATA_DIR) and uses Django lazily
+Post-bootstrap: requires archivebox.config constants and uses Django lazily
 (``from django.db import ...`` inside functions). Not safe to import pre-bootstrap.
 """
 
@@ -17,7 +17,7 @@ from collections.abc import Callable
 from contextlib import contextmanager
 from sqlite3 import OperationalError as SQLiteOperationalError
 
-from archivebox.config import DATA_DIR
+from archivebox.config import CONSTANTS
 from archivebox.misc.util import enforce_types
 
 
@@ -79,7 +79,7 @@ def run_db_analyze_batch(
         return []
 
     next_table, *rest = remaining
-    raw_conn = getattr(connection, "connection", None)
+    raw_conn = connection.connection
     progress_handler_set = False
     if raw_conn is not None and max_seconds_per_table > 0:
         deadline = time.monotonic() + max_seconds_per_table
@@ -117,7 +117,7 @@ def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
     return " ".join([Path(parts[0]).name, *parts[1:]])[:220]
 
 
-def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]:
+def sqlite_lock_holders(db_path: Path = CONSTANTS.DATABASE_FILE) -> list[str]:
     import psutil
 
     db_path = db_path.resolve()
@@ -146,7 +146,7 @@ def sqlite_lock_holders(db_path: Path = DATA_DIR / "index.sqlite3") -> list[str]
     return holders
 
 
-def log_sqlite_lock_holders(console: Any, *, db_path: Path = DATA_DIR / "index.sqlite3", limit: int = 8) -> None:
+def log_sqlite_lock_holders(console: Any, *, db_path: Path = CONSTANTS.DATABASE_FILE, limit: int = 8) -> None:
     holders = sqlite_lock_holders(db_path)
     if holders:
         console.print("[yellow]    DB holders:[/yellow]")
@@ -346,7 +346,7 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
 
 
 @enforce_types
-def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
+def migration_state(out_dir: Path = CONSTANTS.DATA_DIR) -> tuple[list[str], list[str], dict[str, str]]:
     """Cheaply compare migration files to django_migrations without invoking migrate."""
     from django.apps import apps
     from django.db import connection
@@ -373,7 +373,7 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
     loader.load_disk()
     for (app_label, migration_name), migration in loader.disk_migrations.items():
         disk_migrations.add((app_label, migration_name))
-        for replaced_app, replaced_name in getattr(migration, "replaces", ()) or ():
+        for replaced_app, replaced_name in migration.replaces or ():
             squashed_replaced.add((replaced_app, replaced_name))
 
     applied = {(app, name) for app, name in applied if app in app_labels}
@@ -392,14 +392,19 @@ def migration_state(out_dir: Path = DATA_DIR) -> tuple[list[str], list[str], dic
 
 
 @enforce_types
-def pending_migrations(out_dir: Path = DATA_DIR) -> list[str]:
+def pending_migrations(out_dir: Path = CONSTANTS.DATA_DIR) -> list[str]:
     """Return migration files on disk that have not been applied yet."""
     pending, _missing_from_code, _rollback_targets = migration_state(out_dir=out_dir)
     return pending
 
 
 @enforce_types
-def apply_migrations(out_dir: Path = DATA_DIR, stdout: TextIO | None = None, stderr: TextIO | None = None, verbosity: int = 1) -> list[str]:
+def apply_migrations(
+    out_dir: Path = CONSTANTS.DATA_DIR,
+    stdout: TextIO | None = None,
+    stderr: TextIO | None = None,
+    verbosity: int = 1,
+) -> list[str]:
     """Apply pending Django migrations"""
     from django.core.management import call_command
 

archivebox/misc/logging_util.py

@@ -1,7 +1,7 @@
 __package__ = "archivebox"
 
 # Post-bootstrap CLI logging helpers (event loggers, progress bars, formatters).
-# Requires archivebox.config to be loaded — imports DATA_DIR, get_config, and
+# Requires archivebox.config to be loaded — imports CONSTANTS/get_config and
 # references Django ORM types. For pre-bootstrap logging primitives use
 # misc/logging.py, which has no archivebox or Django dependencies.
 
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
 from rich import print
 from rich.panel import Panel
 
-from archivebox.config import DATA_DIR, VERSION
+from archivebox.config import CONSTANTS, VERSION
 from archivebox.config.common import get_config
 from archivebox.misc.util import enforce_types
 from archivebox.misc.logging import ANSI
@@ -171,7 +171,9 @@ def log_list_finished(snapshots):
 
 
 def log_removal_started(snapshots, yes: bool):
-    count = snapshots.count() if hasattr(snapshots, "count") else len(snapshots)
+    from django.db.models import QuerySet
+
+    count = snapshots.count() if isinstance(snapshots, QuerySet) else len(snapshots)
     print(f"[yellow3][i] Found {count} matching URLs to remove.[/]")
     file_counts = [s.num_outputs for s in snapshots if os.access(s.output_dir, os.R_OK)]
     print(
@@ -203,7 +205,7 @@ def log_removal_finished(remaining_links: int, removed_links: int):
 
 
 @enforce_types
-def pretty_path(path: Path | str, pwd: Path | str = DATA_DIR, color: bool = True) -> str:
+def pretty_path(path: Path | str, pwd: Path | str = CONSTANTS.DATA_DIR, color: bool = True) -> str:
     """convert paths like .../ArchiveBox/archivebox/../output/abc into output/abc"""
     pwd = str(Path(pwd))  # .resolve()
     path = str(path)

archivebox/misc/paginators.py

@@ -4,6 +4,7 @@ from django.core.paginator import Paginator
 from django.core.paginator import Page
 from django.core.paginator import EmptyPage
 from django.db import connection
+from django.db.models import QuerySet
 from django.utils.functional import cached_property
 
 
@@ -19,9 +20,13 @@ class CountlessPage(Page):
 class CountlessPaginator(Paginator):
     has_exact_count = False
 
+    def __init__(self, *args, **kwargs):
+        self._count_hint = 0
+        super().__init__(*args, **kwargs)
+
     @cached_property
     def count(self):
-        return getattr(self, "_count_hint", 0)
+        return self._count_hint
 
     @cached_property
     def num_pages(self):
@@ -56,16 +61,29 @@ class AcceleratedPaginator(Paginator):
 
     @cached_property
     def count(self):
-        query = getattr(self.object_list, "query", None)
-        if query is not None and (getattr(query, "distinct", False) or getattr(getattr(query, "where", None), "children", None)):
+        if not isinstance(self.object_list, QuerySet):
+            return super().count
+
+        query = self.object_list.query
+        count_hint = self.object_list.__dict__.get("_archivebox_count_hint")
+        if count_hint is None:
+            count_hint = query.__dict__.get("_archivebox_count_hint")
+        if count_hint is not None:
+            model = self.object_list.model
+            if count_hint == "model_estimate":
+                return self._model_count_estimate(model)
+            if callable(count_hint):
+                return count_hint()
+            return count_hint
+
+        if query.distinct or query.where.children:
             # fallback to normal count method on filtered queryset
             return super().count
 
-        model = getattr(self.object_list, "model", None)
-        if model is None:
-            return super().count
-
         # otherwise count total rows in a separate fast query
+        return self._model_count_estimate(self.object_list.model)
+
+    def _model_count_estimate(self, model):
         if connection.vendor == "sqlite":
             table_name = model._meta.db_table
             with connection.cursor() as cursor:

archivebox/misc/serve_static.py

@@ -2,6 +2,7 @@ import html
 import json
 import re
 import os
+import sys
 import stat
 import asyncio
 import posixpath
@@ -17,6 +18,7 @@ from pathlib import Path
 from urllib.parse import urlencode
 
 from django import template
+from django.core.handlers.asgi import ASGIRequest
 from django.contrib.staticfiles import finders
 from django.template import TemplateDoesNotExist, loader
 from django.views import static
@@ -117,7 +119,7 @@ def _render_mhtml_preview_document(filename: str, output_path: str) -> str:
 
 
 def _format_direntry_timestamp(stat_result: os.stat_result) -> str:
-    timestamp = getattr(stat_result, "st_birthtime", None) or stat_result.st_mtime
+    timestamp = stat_result.st_birthtime if sys.platform == "darwin" else stat_result.st_mtime
     return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M")
 
 
@@ -333,7 +335,7 @@ mimetypes.add_type("multipart/related", ".mhtml")
 mimetypes.add_type("multipart/related", ".mht")
 
 try:
-    _markdown = getattr(importlib.import_module("markdown"), "markdown")
+    _markdown = importlib.import_module("markdown").markdown
 except ImportError:
     _markdown: Callable[..., str] | None = None
 
@@ -716,10 +718,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     https://github.com/satchamo/django/commit/2ce75c5c4bee2a858c0214d136bfcd351fcde11d
     """
     assert document_root
-    config = getattr(request, "archivebox_config", None)
-    if config is None:
-        config = get_config(resolve_plugins=False)
-        request.archivebox_config = config
+    config = request.archivebox_config
     fullpath, path = _resolve_archive_path(document_root, path)
     if os.access(fullpath, os.R_OK) and fullpath.is_dir():
         if request.GET.get("download") == "zip" and show_indexes:
@@ -727,7 +726,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 fullpath,
                 path,
                 is_archive_replay=is_archive_replay,
-                use_async_stream=hasattr(request, "scope"),
+                use_async_stream=isinstance(request, ASGIRequest),
                 config=config,
             )
         if show_indexes:
@@ -872,7 +871,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
                 fullpath.name,
                 raw_output_path,
                 wacz_path=fullpath,
-                fallback_url=getattr(request, "archivebox_snapshot_url", "") or "",
+                fallback_url=request.archivebox_snapshot_url or "",
                 last_modified=http_date(statobj.st_mtime),
                 etag=etag or "",
                 cache_control=(
@@ -978,7 +977,7 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
     # setup response object
     ranged_file = RangedFileReader(open(fullpath, "rb"))
     response = StreamingHttpResponse(
-        _stream_ranged_file_async(ranged_file) if hasattr(request, "scope") else ranged_file,
+        _stream_ranged_file_async(ranged_file) if isinstance(request, ASGIRequest) else ranged_file,
         content_type=content_type,
     )
     response.headers["Last-Modified"] = http_date(statobj.st_mtime)

archivebox/misc/toml_util.py

@@ -96,7 +96,7 @@ class JSONSchemaWithLambdas(GenerateJsonSchema):
 
 def better_toml_dump_str(val: Any) -> str:
     try:
-        dump_str = cast(Callable[[Any], str], getattr(toml.encoder, "_dump_str"))
+        dump_str = cast(Callable[[Any], str], toml.encoder._dump_str)
         return dump_str(val)
     except Exception:
         # if we hit any of toml's numerous encoding bugs,

archivebox/misc/util.py

@@ -474,7 +474,7 @@ class ExtendedEncoder(pyjson.JSONEncoder):
     def default(self, o):
         cls_name = o.__class__.__name__
 
-        if hasattr(o, "_asdict"):
+        if isinstance(o, tuple) and "_asdict" in vars(type(o)):
             return o._asdict()
 
         elif isinstance(o, bytes):

archivebox/personas/migrations/0004_hydrate_persona_permissions.py

@@ -1,5 +1,6 @@
 import os
 import json
+import uuid
 
 from django.db import migrations
 from django.db.models import Q
@@ -9,31 +10,6 @@ VALID_PERMISSIONS = {"public", "unlisted", "private"}
 BATCH_SIZE = 1000
 
 
-def legacy_bool(value):
-    if value is None:
-        return None
-    normalized = str(value).strip().lower()
-    if normalized in {"1", "true", "yes", "on"}:
-        return True
-    if normalized in {"0", "false", "no", "off"}:
-        return False
-    return None
-
-
-def permissions_from_legacy_public_flags(config):
-    if str(config.get("PERMISSIONS") or "").strip():
-        return None
-    public_snapshots = legacy_bool(config.get("PUBLIC_SNAPSHOTS"))
-    public_index = legacy_bool(config.get("PUBLIC_INDEX"))
-    if public_snapshots is False:
-        return "private"
-    if public_index is False:
-        return "unlisted"
-    if public_snapshots is True or public_index is True:
-        return "public"
-    return None
-
-
 def normalize_permissions(value, default):
     value = str(value or "").strip().lower()
     return value if value in VALID_PERMISSIONS else default
@@ -59,22 +35,19 @@ def raw_base_config(apps):
 
 
 def resolve_permissions(config, default):
+    from archivebox.config.common import permissions_from_legacy_public_flags
+
     explicit = str(config.get("PERMISSIONS") or "").strip().lower()
     if explicit in VALID_PERMISSIONS:
         return explicit
     return permissions_from_legacy_public_flags(config) or default
 
 
-def model_has_config(model):
-    try:
-        model._meta.get_field("config")
-    except Exception:
-        return False
-    return True
-
-
 def id_values(pk):
-    return str(pk), getattr(pk, "hex", str(pk).replace("-", ""))
+    if isinstance(pk, uuid.UUID):
+        return str(pk), pk.hex
+    pk_str = str(pk)
+    return pk_str, pk_str.replace("-", "")
 
 
 def flush_batch(cursor, table_name, batch):
@@ -120,8 +93,6 @@ def _ensure_permissions_column(cursor):
 
 def hydrate_persona_permissions(apps, schema_editor):
     Persona = apps.get_model("personas", "Persona")
-    User = apps.get_model("auth", "User")
-    user_has_config = model_has_config(User)
     base_config = raw_base_config(apps)
     default_permissions = resolve_permissions(base_config, "public")
     table_name = schema_editor.quote_name(Persona._meta.db_table)
@@ -130,13 +101,9 @@ def hydrate_persona_permissions(apps, schema_editor):
     batch = []
     missing_permissions = Q(permissions__isnull=True) | (Q(permissions__isnull=False) & ~Q(permissions__in=VALID_PERMISSIONS))
 
-    for persona in Persona.objects.filter(missing_permissions).select_related("created_by").iterator(chunk_size=BATCH_SIZE):
+    for persona in Persona.objects.filter(missing_permissions).iterator(chunk_size=BATCH_SIZE):
         config = dict(persona.config or {})
         resolved = dict(base_config)
-        if user_has_config:
-            user_config = persona.created_by.config or {}
-            if isinstance(user_config, dict):
-                resolved.update(user_config)
         resolved.update(config)
         config["PERMISSIONS"] = resolve_permissions(resolved, default_permissions)
         batch.append((persona.id, config))

archivebox/personas/models.py

@@ -16,6 +16,7 @@ import sys
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
+from collections.abc import Mapping
 
 from django.db import models
 from django.db.models.fields.json import KT
@@ -59,6 +60,26 @@ VOLATILE_PROFILE_FILE_NAMES = {
 }
 
 
+def derive_persona_config(*, name: str, config: Mapping[str, Any] | None, persona_dir: Path) -> dict[str, Any]:
+    derived = dict(config or {})
+
+    if "CHROME_USER_DATA_DIR" not in derived:
+        derived["CHROME_USER_DATA_DIR"] = str(persona_dir / "chrome_profile")
+    if "CHROME_DOWNLOADS_DIR" not in derived:
+        derived["CHROME_DOWNLOADS_DIR"] = str(persona_dir / "chrome_downloads")
+
+    cookies_path = persona_dir / "cookies.txt"
+    if "COOKIES_FILE" not in derived and cookies_path.exists():
+        derived["COOKIES_FILE"] = str(cookies_path)
+
+    auth_path = persona_dir / "auth.json"
+    if "AUTH_STORAGE_FILE" not in derived and auth_path.exists():
+        derived["AUTH_STORAGE_FILE"] = str(auth_path)
+
+    derived["ACTIVE_PERSONA"] = name
+    return derived
+
+
 class Persona(ModelWithConfig):
     """
     Browser persona/profile for archiving sessions.
@@ -99,8 +120,7 @@ class Persona(ModelWithConfig):
         if str(config.get("PERMISSIONS") or "").strip().lower() not in PERMISSIONS_VALUES:
             from archivebox.config.common import get_config
 
-            user = self.created_by if self.created_by_id else None
-            config["PERMISSIONS"] = normalize_permissions(get_config(user=user, include_machine=True).PERMISSIONS)
+            config["PERMISSIONS"] = normalize_permissions(get_config(include_machine=True).PERMISSIONS)
             self.config = config
             update_fields = kwargs.get("update_fields")
             if update_fields is not None:
@@ -151,22 +171,7 @@ class Persona(ModelWithConfig):
         - AUTH_STORAGE_FILE (derived from persona path, if file exists)
         - ACTIVE_PERSONA (set to this persona's name)
         """
-        derived = dict(self.config or {})
-
-        # Add derived paths (don't override if explicitly set in config)
-        if "CHROME_USER_DATA_DIR" not in derived:
-            derived["CHROME_USER_DATA_DIR"] = self.CHROME_USER_DATA_DIR
-        if "CHROME_DOWNLOADS_DIR" not in derived:
-            derived["CHROME_DOWNLOADS_DIR"] = self.CHROME_DOWNLOADS_DIR
-        if "COOKIES_FILE" not in derived and self.COOKIES_FILE:
-            derived["COOKIES_FILE"] = self.COOKIES_FILE
-        if "AUTH_STORAGE_FILE" not in derived and self.AUTH_STORAGE_FILE:
-            derived["AUTH_STORAGE_FILE"] = self.AUTH_STORAGE_FILE
-
-        # Always set ACTIVE_PERSONA to this persona's name
-        derived["ACTIVE_PERSONA"] = self.name
-
-        return derived
+        return derive_persona_config(name=self.name, config=self.config, persona_dir=self.path)
 
     def ensure_dirs(self) -> None:
         """Create persona directories if they don't exist."""
@@ -232,6 +237,15 @@ class Persona(ModelWithConfig):
     def runtime_downloads_dir_for_crawl(self, crawl) -> Path:
         return self.runtime_root_for_crawl(crawl) / "chrome_downloads"
 
+    def runtime_root_for_snapshot(self, snapshot) -> Path:
+        return Path(snapshot.output_dir) / ".persona" / self.name
+
+    def runtime_profile_dir_for_snapshot(self, snapshot) -> Path:
+        return self.runtime_root_for_snapshot(snapshot) / "chrome_profile"
+
+    def runtime_downloads_dir_for_snapshot(self, snapshot) -> Path:
+        return self.runtime_root_for_snapshot(snapshot) / "chrome_downloads"
+
     def copy_chrome_profile(self, source_dir: Path, destination_dir: Path) -> None:
         destination_dir.parent.mkdir(parents=True, exist_ok=True)
         shutil.rmtree(destination_dir, ignore_errors=True)
@@ -284,6 +298,33 @@ class Persona(ModelWithConfig):
             "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
         }
 
+    def prepare_runtime_for_snapshot(self, snapshot, chrome_binary: str = "") -> dict[str, str]:
+        crawl_runtime_profile_dir = self.runtime_profile_dir_for_crawl(snapshot.crawl)
+        template_dir = crawl_runtime_profile_dir if crawl_runtime_profile_dir.exists() else Path(self.CHROME_USER_DATA_DIR)
+        runtime_root = self.runtime_root_for_snapshot(snapshot)
+        runtime_profile_dir = self.runtime_profile_dir_for_snapshot(snapshot)
+        runtime_downloads_dir = self.runtime_downloads_dir_for_snapshot(snapshot)
+
+        if runtime_root.exists():
+            shutil.rmtree(runtime_root, ignore_errors=True)
+        if template_dir.exists() and any(template_dir.iterdir()):
+            self.copy_chrome_profile(template_dir, runtime_profile_dir)
+        else:
+            runtime_profile_dir.mkdir(parents=True, exist_ok=True)
+
+        runtime_downloads_dir.mkdir(parents=True, exist_ok=True)
+        self.cleanup_chrome_profile(runtime_profile_dir)
+
+        (runtime_root / "persona_name.txt").write_text(self.name)
+        (runtime_root / "template_dir.txt").write_text(str(template_dir))
+        if chrome_binary:
+            (runtime_root / "chrome_binary.txt").write_text(chrome_binary)
+
+        return {
+            "CHROME_USER_DATA_DIR": str(runtime_profile_dir),
+            "CHROME_DOWNLOADS_DIR": str(runtime_downloads_dir),
+        }
+
     def cleanup_runtime_for_crawl(self, crawl) -> None:
         shutil.rmtree(Path(crawl.output_dir) / ".persona", ignore_errors=True)
 

archivebox/plugins/discovery.py

@@ -1,7 +1,6 @@
 __package__ = "archivebox.plugins"
 
 import json
-import os
 from collections.abc import Iterable
 from functools import lru_cache
 from pathlib import Path
@@ -26,9 +25,7 @@ class PluginSpecialConfig(TypedDict):
 
 
 BUILTIN_PLUGINS_DIR = Path(get_plugins_dir()).resolve()
-USER_PLUGINS_DIR = Path(
-    os.environ.get("ARCHIVEBOX_USER_PLUGINS_DIR") or str(CONSTANTS.USER_PLUGINS_DIR),
-).expanduser()
+USER_PLUGINS_DIR = CONSTANTS.USER_PLUGINS_DIR
 
 
 def iter_plugin_dirs() -> list[Path]:
@@ -167,7 +164,7 @@ def discover_plugins_that_provide_interface(
                 module = importlib.util.module_from_spec(spec)
                 spec.loader.exec_module(module)
 
-                if not all(hasattr(module, attr) for attr in required_attrs):
+                if not all(attr in vars(module) for attr in required_attrs):
                     continue
 
                 if plugin_prefix:
@@ -198,12 +195,15 @@ def get_search_backends() -> dict[str, Any]:
     )
 
 
+@lru_cache(maxsize=1)
 def discover_plugin_configs() -> dict[str, dict[str, Any]]:
     """
     Discover all plugin config.json schemas.
 
     Each plugin can define a config.json file with JSONSchema defining
-    its configuration options.
+    its configuration options. This is intentionally cached because these
+    schemas are plugin package metadata, not live user config; runtime values
+    still come from env/db config at each callsite.
     """
     configs = {}
 

archivebox/plugins/forms.py

@@ -9,7 +9,7 @@ from typing import Any
 from django import forms
 from django.utils.html import format_html
 
-from archivebox.config.common import get_config
+from archivebox.config.common import ArchiveBoxConfig, get_config
 from archivebox.plugins.discovery import discover_plugin_configs, get_plugin_icon, get_plugins
 
 
@@ -250,6 +250,7 @@ def _coerce_plugin_config_value(raw_value: Any, schema: Mapping[str, Any]) -> An
 
 class PluginConfigFormMixin:
     plugin_groups: list[dict[str, Any]]
+    allow_crawl_execution_config_fields = True
 
     def build_plugin_groups(self, runtime_config: Mapping[str, Any] | None = None) -> None:
         all_plugins = get_plugins()
@@ -314,7 +315,10 @@ class PluginConfigFormMixin:
             config_fields = [
                 self._build_plugin_config_field(str(plugin_name), str(config_key), prop_schema, runtime_config)
                 for config_key, prop_schema in properties.items()
-                if isinstance(prop_schema, dict)
+                if (
+                    isinstance(prop_schema, dict)
+                    and (self.allow_crawl_execution_config_fields or ArchiveBoxConfig.scope_for_key(str(config_key)) == "crawl_frozen")
+                )
             ]
             cards.append(
                 {
@@ -427,6 +431,8 @@ class PluginConfigFormMixin:
                 input_name = _plugin_config_input_name(plugin_name, config_key)
                 if input_name not in self.data:
                     continue
+                if not self.allow_crawl_execution_config_fields and ArchiveBoxConfig.scope_for_key(str(config_key)) != "crawl_frozen":
+                    continue
 
                 raw_value: Any = self.data.get(input_name)
                 if "array" in _schema_types(prop_schema) and isinstance(prop_schema.get("enum"), list):
@@ -595,5 +601,6 @@ def get_plugin_config_binary_urls(runtime_config: Mapping[str, Any]) -> dict[str
             )
         if binary is None and name != value:
             binary = Binary.objects.get_valid_binary(name, machine=machine)
-        urls[key] = get_installed_binary_change_url(getattr(binary, "name", name), binary) or get_environment_binary_url(name)
+        binary_name = binary.name if binary is not None else name
+        urls[key] = get_installed_binary_change_url(binary_name, binary) or get_environment_binary_url(name)
     return urls

archivebox/plugins/hooks.py

@@ -46,7 +46,7 @@ import json
 import os
 from collections.abc import Mapping
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard
+from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeGuard, runtime_checkable
 
 from archivebox.config.constants import CONSTANTS
 from archivebox.config.version import VERSION
@@ -62,12 +62,13 @@ if TYPE_CHECKING:
     from archivebox.machine.models import Process
 
 
+@runtime_checkable
 class ConfigDump(Protocol):
     def as_dict(self) -> dict[str, Any]: ...
 
 
 def _has_config_dump(config: object) -> TypeGuard[ConfigDump]:
-    return callable(getattr(config, "as_dict", None))
+    return isinstance(config, ConfigDump)
 
 
 def _config_to_overrides(config: ConfigLookup | Mapping[str, Any] | None) -> dict[str, Any]:
@@ -259,13 +260,13 @@ def run_hook(
     This is the low-level hook executor that creates a Process record and
     uses Process.launch() for subprocess management.
 
-    Config is passed to hooks via environment variables. Caller MUST use
-    get_config() to merge all sources (file, env, machine, crawl, snapshot).
+    Config is passed to hooks via environment variables. Crawl/snapshot callers
+    should pass the runtime config produced by for_crawl_runtime().
 
     Args:
         script: Path to the hook script (.sh, .py, or .js)
         output_dir: Working directory for the script (where output files go)
-        config: Optional pre-merged config dict from get_config(crawl=..., snapshot=...).
+        config: Optional runtime config dict from for_crawl_runtime().
                 If omitted, pass scope/override args using kwargs prefixed with config_.
         timeout: Maximum execution time in seconds
                  If None, auto-detects from PLUGINNAME_TIMEOUT config (fallback to TIMEOUT, default 300)
@@ -277,18 +278,22 @@ def run_hook(
 
     Example:
         from archivebox.config.common import get_config
-        config = get_config(crawl=my_crawl, snapshot=my_snapshot)
+        config = get_config(crawl=my_crawl, snapshot=my_snapshot).for_crawl_runtime(crawl=my_crawl, snapshot=my_snapshot)
         process = run_hook(hook_path, output_dir, config=config, url=url, snapshot_id=id)
         if process.status == 'exited':
             records = process.get_records()  # Get parsed JSONL output
     """
     from archivebox.machine.models import Process, Machine, NetworkInterface
-    from archivebox.config.common import get_config
+    from archivebox.config.common import get_config, normalize_runtime_config
     import sys
 
     config_scope = {key.removeprefix("config_"): kwargs.pop(key) for key in list(kwargs) if key.startswith("config_")}
-    resolved_config = get_config(overrides=_config_to_overrides(config), **config_scope)
-    hook_config = resolved_config.for_crawl_execution()
+    config_overrides = _config_to_overrides(config)
+    resolved_config = get_config(overrides=config_overrides, **config_scope)
+    hook_config = normalize_runtime_config(
+        config_overrides if config is not None else resolved_config.for_crawl(),
+        json_safe=False,
+    )
 
     # Auto-detect timeout from plugin config if not explicitly provided
     if timeout is None:
@@ -363,31 +368,16 @@ def run_hook(
 
     # Set up environment with base paths
     env = os.environ.copy()
-    env["DATA_DIR"] = str(resolved_config.DATA_DIR)
-    env["ARCHIVE_DIR"] = str(resolved_config.ARCHIVE_DIR)
-    env["ABX_RUNTIME"] = "archivebox"
+    env["DATA_DIR"] = str(CONSTANTS.DATA_DIR)
     env["LIBRARY_VERSION"] = VERSION
     env.setdefault("MACHINE_ID", os.environ.get("MACHINE_ID", CONSTANTS.MACHINE_ID))
-
-    resolved_output_dir = output_dir.resolve()
-    snap_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
-    crawl_dir = _model_output_dir_from_child_path(resolved_output_dir, CONSTANTS.CRAWLS_DIR_NAME)
+    snap_dir = hook_config.get("SNAP_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.SNAPSHOTS_DIR_NAME)
+    crawl_dir = hook_config.get("CRAWL_DIR") or _model_output_dir_from_child_path(output_dir, CONSTANTS.CRAWLS_DIR_NAME)
     if snap_dir:
         env["SNAP_DIR"] = str(snap_dir)
     if crawl_dir:
         env["CRAWL_DIR"] = str(crawl_dir)
 
-    crawl_id = kwargs.get("_crawl_id") or kwargs.get("crawl_id")
-    if crawl_id:
-        try:
-            from archivebox.crawls.models import Crawl
-
-            crawl = Crawl.objects.filter(id=crawl_id).first()
-            if crawl:
-                env["CRAWL_DIR"] = str(crawl.output_dir)
-        except Exception:
-            pass
-
     # Export runtime library roots; abx-dl/abxpkg own executable lookup env.
     lib_dir = resolved_config.LIB_DIR
     if lib_dir:
@@ -423,7 +413,6 @@ def run_hook(
         "NODE_MODULES_DIR",
         "NODE_MODULE_DIR",
         "DATA_DIR",
-        "ARCHIVE_DIR",
         "MACHINE_ID",
         "SNAP_DIR",
         "CRAWL_DIR",

archivebox/plugins/views.py

@@ -82,7 +82,7 @@ def get_machine_admin_url() -> str | None:
         from archivebox.machine.models import Machine
 
         machine = Machine.current()
-        return getattr(machine, "admin_change_url", None) or f"/admin/machine/machine/{machine.id}/change/"
+        return machine.admin_change_url or f"/admin/machine/machine/{machine.id}/change/"
     except Exception:
         return None
 

archivebox/progressmonitor/views.py

@@ -345,9 +345,7 @@ def live_progress_view(request):
         persona_objects_by_id = {}
         persona_objects_by_name = {}
         persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
-        persona_names = {
-            str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default") for crawl in active_crawls_list if not crawl["persona_id"]
-        }
+        persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
         if persona_ids or persona_names:
             from archivebox.personas.models import Persona
 
@@ -813,7 +811,7 @@ def live_progress_view(request):
             urls_preview = crawl["urls"][:60] if crawl["urls"] else None
             crawl_tags = [tag.strip() for tag in (crawl["tags_str"] or "").replace("\n", ",").split(",") if tag.strip()]
             persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
-            persona_name = persona_details["name"] if persona_details else str((crawl["config"] or {}).get("DEFAULT_PERSONA") or "Default")
+            persona_name = persona_details["name"] if persona_details else "Default"
             persona_details = persona_details or persona_details_by_name.get(persona_name)
             crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
             avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0

archivebox/search/admin.py

@@ -19,8 +19,8 @@ class SearchResultsChangeList(ChangeList):
 
     def __init__(self, request, *args, **kwargs):
         """Capture normalized search mode before Django builds results."""
-        self.search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
-        self.search_mode_backend = get_search_mode_backend(self.search_mode, config=getattr(request, "archivebox_config", None))
+        self.search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
+        self.search_mode_backend = get_search_mode_backend(self.search_mode, config=request.archivebox_config)
         super().__init__(request, *args, **kwargs)
         self.embedded_changelist = request.GET.get("_embedded") == "crawl"
 
@@ -31,7 +31,7 @@ class SearchResultsChangeList(ChangeList):
             self.opts.model_name == "snapshot"
             and self.query
             and self.result_count == 0
-            and get_search_mode_base(self.search_mode, config=getattr(request, "archivebox_config", None)) == "deep"
+            and get_search_mode_base(self.search_mode, config=request.archivebox_config) == "deep"
             and self.search_mode_backend,
         )
 
@@ -55,13 +55,11 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
 
     def get_default_search_mode(self):
         """Return the default search mode for the current request config."""
-        request = getattr(self, "request", None)
-        return get_default_search_mode(config=getattr(request, "archivebox_config", None))
+        return get_default_search_mode(config=self.request.archivebox_config)
 
     def get_search_mode_options(self):
         """Return selector options for the current request config."""
-        request = getattr(self, "request", None)
-        return get_search_mode_options(config=getattr(request, "archivebox_config", None))
+        return get_search_mode_options(config=self.request.archivebox_config)
 
     def get_search_results(self, request, queryset, search_term: str):
         """Apply admin search semantics to a changelist queryset."""
@@ -69,14 +67,14 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
         search_term = search_term.strip()
         if not search_term:
             return super().get_search_results(request, queryset, search_term)
-        search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+        search_mode = get_search_mode(request.GET.get("search_mode"), config=request.archivebox_config)
         if queryset.model._meta.label_lower == "core.snapshot" and request.GET.get("_embedded") != "crawl":
             cached_ids = get_cached_admin_search_ids(request)
             if cached_ids is not None:
                 return queryset.filter(pk__in=cached_ids) if cached_ids else queryset.none(), False
             return queryset.none(), False
 
-        if get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None)) == "meta":
+        if get_search_mode_base(search_mode, config=request.archivebox_config) == "meta":
             qs, use_distinct = super().get_search_results(request, queryset, search_term)
             return qs, use_distinct
         if request.GET.get("_embedded") == "crawl":
@@ -85,7 +83,7 @@ class SearchResultsAdminMixin(admin.ModelAdmin):
                     pk__in=query_search_index(
                         search_term,
                         search_mode=search_mode,
-                        config=getattr(request, "archivebox_config", None),
+                        config=request.archivebox_config,
                     ).values("pk"),
                 ), False
             except Exception as err:

archivebox/search/backends.py

@@ -12,16 +12,16 @@ _search_backends_cache: dict | None = None
 
 @contextmanager
 def search_backend_env(config: dict[str, Any] | None = None, **config_kwargs: Any):
-    """Temporarily expose resolved config through os.environ for backend code."""
+    """Temporarily expose resolved search config through os.environ for backend code."""
     config = config or get_config(**config_kwargs)
     updates = {}
     for key, value in config.items():
+        if not str(key).startswith("SEARCH_BACKEND_"):
+            continue
         if value is None:
             continue
         if isinstance(value, (str, int, float, bool, os.PathLike)):
             updates[str(key)] = str(value)
-    updates["DATA_DIR"] = str(config.DATA_DIR)
-    updates["SNAP_DIR"] = str(config.USERS_DIR)
     previous = {key: os.environ.get(key) for key in updates}
     os.environ.update(updates)
     try:

archivebox/search/query.py

@@ -248,10 +248,8 @@ def iter_query_search_ids(
             backend = backends[backend_name]
             try:
                 with search_backend_env(config=config):
-                    if hasattr(backend, "iter_search"):
+                    if backend_name == "ripgrep":
                         ids = backend.iter_search(query, search_mode=search_mode_base)
-                    elif backend_name == "ripgrep":
-                        ids = backend.search(query, search_mode=search_mode_base)
                     else:
                         ids = backend.search(query)
                     for snapshot_id in ids:

archivebox/search/views.py

@@ -101,7 +101,8 @@ def iter_admin_backend_search_ids(iterator, queryset):
 def admin_snapshot_search_stream_view(model_admin, request):
     """Stream admin Snapshot search progress and cache matching IDs."""
     query = (request.GET.get("q") or "").strip()
-    search_mode = get_search_mode(request.GET.get("search_mode"), config=getattr(request, "archivebox_config", None))
+    config = request.archivebox_config
+    search_mode = get_search_mode(request.GET.get("search_mode"), config=config)
     if not query:
         return StreamingHttpResponse((), content_type="text/plain")
 
@@ -115,12 +116,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
     filter_request.path = target_url.path or request.path
     filter_request.path_info = target_url.path or request.path_info
     filter_request.GET = target_get
-    filter_request.archivebox_config = getattr(request, "archivebox_config", None)
+    filter_request.archivebox_config = config
 
     # Build the same filtered base queryset the changelist uses, but with the
     # search params stripped. The stream intersects each wave with this queryset
     # before writing IDs into the short-lived cache consumed by the changelist.
-    current_request = getattr(model_admin, "request", None)
+    current_request = model_admin.__dict__.get("request")
     try:
         base_queryset = model_admin.get_changelist_instance(filter_request).queryset
     finally:
@@ -150,12 +151,12 @@ def admin_snapshot_search_stream_view(model_admin, request):
             nonlocal last_sent
             iterator = None
             try:
-                search_mode_base = get_search_mode_base(search_mode, config=getattr(request, "archivebox_config", None))
+                search_mode_base = get_search_mode_base(search_mode, config=config)
                 iterator = (
                     iter_admin_meta_search_ids(query, base_queryset)
                     if search_mode_base == "meta"
                     else iter_admin_backend_search_ids(
-                        iter_query_search_ids(query, search_mode=search_mode, config=getattr(request, "archivebox_config", None)),
+                        iter_query_search_ids(query, search_mode=search_mode, config=config),
                         base_queryset,
                     )
                 )

archivebox/services/archive_result_service.py

@@ -2,10 +2,15 @@ from __future__ import annotations
 
 import asyncio
 import json
+import os
+import sys
+import time
 from collections import defaultdict
 from collections.abc import Iterable
+from contextlib import contextmanager
+from functools import wraps
 from pathlib import Path
-from typing import Any
+from typing import Any, Protocol, runtime_checkable
 
 from asgiref.sync import sync_to_async
 from django.db import IntegrityError
@@ -18,6 +23,58 @@ from abx_dl.services.base import BaseService
 from .process_service import parse_event_datetime
 
 
+def _perf_trace(label):
+    def decorator(func):
+        if asyncio.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                    return await func(*args, **kwargs)
+                started_at = time.perf_counter()
+                try:
+                    return await func(*args, **kwargs)
+                finally:
+                    elapsed_ms = (time.perf_counter() - started_at) * 1000
+                    print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+            return async_wrapper
+
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                return func(*args, **kwargs)
+            started_at = time.perf_counter()
+            try:
+                return func(*args, **kwargs)
+            finally:
+                elapsed_ms = (time.perf_counter() - started_at) * 1000
+                print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+        return sync_wrapper
+
+    return decorator
+
+
+@contextmanager
+def _perf_span(label: str):
+    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+        yield
+        return
+    started_at = time.perf_counter()
+    try:
+        yield
+    finally:
+        elapsed_ms = (time.perf_counter() - started_at) * 1000
+        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+
+@runtime_checkable
+class ModelDumpable(Protocol):
+    def model_dump(self) -> dict[str, Any]: ...
+
+
+@_perf_trace("archivebox.ArchiveResultService._collect_output_metadata")
 def _collect_output_metadata(plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
     exclude_names = {"stdout.log", "stderr.log", "process.pid", "hook.pid", "listener.pid"}
     output_files: dict[str, dict] = {}
@@ -97,15 +154,8 @@ def _normalize_output_files(raw_output_files: Any) -> dict[str, dict]:
         if isinstance(item, str):
             normalized[item] = _enrich_metadata(item, {})
             continue
-        if hasattr(item, "model_dump"):
+        if isinstance(item, ModelDumpable):
             item = item.model_dump()
-        elif hasattr(item, "path"):
-            item = {
-                "path": getattr(item, "path", ""),
-                "extension": getattr(item, "extension", ""),
-                "mimetype": getattr(item, "mimetype", ""),
-                "size": getattr(item, "size", 0),
-            }
         if not isinstance(item, dict):
             continue
         path = str(item.get("path") or "").strip()
@@ -137,6 +187,7 @@ def _summarize_output_files(output_files: dict[str, dict]) -> tuple[int, str]:
     return total_size, output_mimetypes
 
 
+@_perf_trace("archivebox.ArchiveResultService._resolve_output_metadata")
 def _resolve_output_metadata(raw_output_files: Any, plugin_dir: Path) -> tuple[dict[str, dict], int, str]:
     normalized_output_files = _normalize_output_files(raw_output_files)
     if normalized_output_files and _has_structured_output_metadata(normalized_output_files):
@@ -217,93 +268,112 @@ class ArchiveResultService(BaseService):
         self.bus.on(ArchiveResultEvent, self.on_ArchiveResultEvent__save_to_db)
         self.bus.on(ProcessCompletedEvent, self.on_ProcessCompletedEvent__save_to_db)
 
+    @_perf_trace("archivebox.ArchiveResultService.on_ArchiveResultEvent__save_to_db")
     async def on_ArchiveResultEvent__save_to_db(self, event: ArchiveResultEvent) -> None:
         from archivebox.core.models import ArchiveResult, Snapshot
         from archivebox.machine.models import Process
 
-        snapshot = await Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").afirst()
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.snapshot_lookup"):
+            snapshot = await Snapshot.objects.filter(id=event.snapshot_id).select_related("crawl", "crawl__created_by").afirst()
         if snapshot is None:
             return
-        plugin_dir = Path(snapshot.output_dir) / event.plugin
-        output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
-        process_started = await self.bus.find(
-            ProcessStartedEvent,
-            past=True,
-            future=False,
-            where=lambda candidate: self.bus.event_is_child_of(event, candidate),
-        )
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.find_process_started"):
+            process_started = await self.bus.find(
+                ProcessStartedEvent,
+                past=True,
+                future=False,
+                where=lambda candidate: self.bus.event_is_child_of(event, candidate),
+            )
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.plugin_dir"):
+            plugin_dir = (
+                Path(process_started.output_dir)
+                if process_started is not None and process_started.output_dir
+                else Path(snapshot.output_dir) / event.plugin
+            )
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.resolve_output_metadata"):
+            output_files, output_size, output_mimetypes = await sync_to_async(_resolve_output_metadata)(event.output_files, plugin_dir)
         process = None
         if process_started is not None:
-            started_at = parse_event_datetime(process_started.start_ts)
-            if started_at is None:
-                raise ValueError("ProcessStartedEvent.start_ts is required")
-            process_query = Process.objects.filter(
-                pwd=process_started.output_dir,
-                cmd=[process_started.hook_path, *process_started.hook_args],
-                started_at=started_at,
-            )
-            if process_started.pid:
-                process_query = process_query.filter(pid=process_started.pid)
-            process = await process_query.order_by("-modified_at").afirst()
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.process_lookup"):
+                started_at = parse_event_datetime(process_started.start_ts)
+                if started_at is None:
+                    raise ValueError("ProcessStartedEvent.start_ts is required")
+                process_query = Process.objects.filter(
+                    pwd=process_started.output_dir,
+                    cmd=[process_started.hook_path, *process_started.hook_args],
+                    started_at=started_at,
+                )
+                if process_started.pid:
+                    process_query = process_query.filter(pid=process_started.pid)
+                process = await process_query.order_by("-modified_at").afirst()
 
-        start_ts = parse_event_datetime(event.start_ts)
-        end_ts = parse_event_datetime(event.end_ts) or timezone.now()
-        defaults = {
-            "status": _normalize_status(event.status),
-            "output_str": event.output_str,
-            "output_json": event.output_json,
-            "output_files": output_files,
-            "output_size": output_size,
-            "output_mimetypes": output_mimetypes,
-            "start_ts": start_ts or timezone.now(),
-            "end_ts": end_ts,
-        }
-        if process is not None:
-            defaults["process_id"] = process.id
-        if event.error:
-            defaults["notes"] = event.error
+        with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.prepare_defaults"):
+            start_ts = parse_event_datetime(event.start_ts)
+            end_ts = parse_event_datetime(event.end_ts) or timezone.now()
+            defaults = {
+                "status": _normalize_status(event.status),
+                "output_str": event.output_str,
+                "output_json": event.output_json,
+                "output_files": output_files,
+                "output_size": output_size,
+                "output_mimetypes": output_mimetypes,
+                "start_ts": start_ts or timezone.now(),
+                "end_ts": end_ts,
+            }
+            if process is not None:
+                defaults["process_id"] = process.id
+            if event.error:
+                defaults["notes"] = event.error
 
         key = (str(snapshot.id), event.plugin, event.hook_name)
         lock = self._save_locks.setdefault(key, asyncio.Lock())
         async with lock:
-            # Avoid update_or_create(): Django wraps it in transaction.atomic().
-            # On SQLite a lock retry inside that read-then-write transaction can
-            # keep the read transaction open and block the writer that would have
-            # released the lock. Use autocommit-sized writes instead.
-            result = await ArchiveResult.objects.filter(
-                snapshot=snapshot,
-                plugin=event.plugin,
-                hook_name=event.hook_name,
-            ).afirst()
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_lookup"):
+                # Avoid update_or_create(): Django wraps it in transaction.atomic().
+                # On SQLite a lock retry inside that read-then-write transaction can
+                # keep the read transaction open and block the writer that would have
+                # released the lock. Use autocommit-sized writes instead.
+                result = await ArchiveResult.objects.filter(
+                    snapshot=snapshot,
+                    plugin=event.plugin,
+                    hook_name=event.hook_name,
+                ).afirst()
             if result is None:
                 try:
-                    result = await ArchiveResult.objects.acreate(
-                        snapshot=snapshot,
-                        plugin=event.plugin,
-                        hook_name=event.hook_name,
-                        **defaults,
-                    )
+                    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_create"):
+                        result = await ArchiveResult.objects.acreate(
+                            snapshot=snapshot,
+                            plugin=event.plugin,
+                            hook_name=event.hook_name,
+                            **defaults,
+                        )
                 except IntegrityError:
-                    result = await ArchiveResult.objects.aget(
-                        snapshot=snapshot,
-                        plugin=event.plugin,
-                        hook_name=event.hook_name,
-                    )
+                    with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_get_after_integrity"):
+                        result = await ArchiveResult.objects.aget(
+                            snapshot=snapshot,
+                            plugin=event.plugin,
+                            hook_name=event.hook_name,
+                        )
 
-            update_fields = []
-            for field, value in defaults.items():
-                if getattr(result, field) != value:
-                    setattr(result, field, value)
-                    update_fields.append(field)
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
+                update_fields = []
+                for field, value in defaults.items():
+                    if result.__dict__[field] != value:
+                        setattr(result, field, value)
+                        update_fields.append(field)
             if update_fields:
-                await result.asave(update_fields=[*update_fields, "modified_at"])
+                with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.result_update"):
+                    await result.asave(update_fields=[*update_fields, "modified_at"])
 
         if result.status in (ArchiveResult.StatusChoices.SUCCEEDED, ArchiveResult.StatusChoices.NORESULTS):
-            next_title = _extract_snapshot_title(str(snapshot.output_dir), event.plugin, result.output_str, snapshot_url=snapshot.url)
-            if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):
-                snapshot.title = next_title
-                await snapshot.asave(update_fields=["title", "modified_at"])
+            with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.title_update"):
+                title_output_str = result.output_str if result.status == ArchiveResult.StatusChoices.SUCCEEDED else ""
+                next_title = _extract_snapshot_title(str(plugin_dir.parent), event.plugin, title_output_str, snapshot_url=snapshot.url)
+                if next_title and _should_update_snapshot_title(snapshot.title or "", next_title, snapshot_url=snapshot.url):
+                    snapshot.title = next_title
+                    await snapshot.asave(update_fields=["title", "modified_at"])
 
+    @_perf_trace("archivebox.ArchiveResultService.on_ProcessCompletedEvent__save_to_db")
     async def on_ProcessCompletedEvent__save_to_db(self, event: ProcessCompletedEvent) -> None:
         if event.event_id in self._completed_process_event_ids:
             return
@@ -311,16 +381,18 @@ class ArchiveResultService(BaseService):
 
         if not event.hook_name.startswith("on_Snapshot"):
             return
-        snapshot_event = await self.bus.find(
-            SnapshotEvent,
-            past=True,
-            future=False,
-            where=lambda candidate: self.bus.event_is_child_of(event, candidate),
-        )
+        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.find_snapshot_event"):
+            snapshot_event = await self.bus.find(
+                SnapshotEvent,
+                past=True,
+                future=False,
+                where=lambda candidate: self.bus.event_is_child_of(event, candidate),
+            )
         if snapshot_event is None:
             return
 
-        records = _iter_archiveresult_records(event.stdout)
+        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.parse_stdout_records"):
+            records = _iter_archiveresult_records(event.stdout)
         if records:
             if len(records) > 1:
                 raise RuntimeError(
@@ -329,33 +401,35 @@ class ArchiveResultService(BaseService):
             for record in records:
                 record_status = _normalize_status(record.get("status") or "")
                 record_failed = record_status == "failed" or (not record_status and event.exit_code not in (0, PROCESS_EXIT_SKIPPED))
-                await event.emit(
-                    ArchiveResultEvent(
-                        snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id,
-                        plugin=record.get("plugin") or event.plugin_name,
-                        hook_name=record.get("hook_name") or event.hook_name,
-                        status=record_status,
-                        output_str=record.get("output_str") or "",
-                        output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None,
-                        output_files=event.output_files,
-                        start_ts=event.start_ts,
-                        end_ts=event.end_ts,
-                        error=record.get("error") or (event.stderr if record_failed else ""),
-                    ),
-                ).now()
+                with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_record"):
+                    await event.emit(
+                        ArchiveResultEvent(
+                            snapshot_id=record.get("snapshot_id") or snapshot_event.snapshot_id,
+                            plugin=record.get("plugin") or event.plugin_name,
+                            hook_name=record.get("hook_name") or event.hook_name,
+                            status=record_status,
+                            output_str=record.get("output_str") or "",
+                            output_json=record.get("output_json") if isinstance(record.get("output_json"), dict) else None,
+                            output_files=event.output_files,
+                            start_ts=event.start_ts,
+                            end_ts=event.end_ts,
+                            error=record.get("error") or (event.stderr if record_failed else ""),
+                        ),
+                    ).now()
             return
 
         process_failed = event.exit_code not in (0, PROCESS_EXIT_SKIPPED)
-        await event.emit(
-            ArchiveResultEvent(
-                snapshot_id=snapshot_event.snapshot_id,
-                plugin=event.plugin_name,
-                hook_name=event.hook_name,
-                status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "skipped"),
-                output_str=event.stderr if process_failed else "",
-                output_files=event.output_files,
-                start_ts=event.start_ts,
-                end_ts=event.end_ts,
-                error=event.stderr if process_failed else "",
-            ),
-        ).now()
+        with _perf_span("archivebox.ArchiveResultService.on_ProcessCompletedEvent.emit_archive_result_fallback"):
+            await event.emit(
+                ArchiveResultEvent(
+                    snapshot_id=snapshot_event.snapshot_id,
+                    plugin=event.plugin_name,
+                    hook_name=event.hook_name,
+                    status="failed" if process_failed else ("succeeded" if _has_content_files(event.output_files) else "skipped"),
+                    output_str=event.stderr if process_failed else "",
+                    output_files=event.output_files,
+                    start_ts=event.start_ts,
+                    end_ts=event.end_ts,
+                    error=event.stderr if process_failed else "",
+                ),
+            ).now()

archivebox/services/machine_service.py

@@ -1,5 +1,9 @@
 from __future__ import annotations
 
+import os
+import sys
+import time
+from contextlib import contextmanager
 from typing import Any
 
 from asgiref.sync import sync_to_async
@@ -10,6 +14,19 @@ from abx_dl.services.base import BaseService
 _BINARY_EVENT_ALLOWED_KEYS = frozenset({"ABX_INSTALL_CACHE"})
 
 
+@contextmanager
+def _perf_span(label: str):
+    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+        yield
+        return
+    started_at = time.perf_counter()
+    try:
+        yield
+    finally:
+        elapsed_ms = (time.perf_counter() - started_at) * 1000
+        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+
 def _is_binary_event_key(key: str) -> bool:
     """``MachineEvent`` projector only ever writes binary-related state.
 
@@ -40,29 +57,34 @@ class MachineService(BaseService):
         self.bus.on(MachineEvent, self.on_MachineEvent__save_to_db)
 
     async def on_MachineEvent__save_to_db(self, event: MachineEvent) -> None:
-        from archivebox.machine.models import Machine, _sanitize_machine_config
-        from archivebox.config.common import get_config
+        from archivebox.machine.models import Machine
 
         if event.config_type != "derived":
             return
 
-        machine = await sync_to_async(Machine.current, thread_sensitive=True)()
-        lib_dir = await sync_to_async(lambda: get_config(include_machine=False).LIB_DIR, thread_sensitive=True)()
-        config = dict(machine.config or {})
+        with _perf_span("archivebox.MachineService.on_MachineEvent.machine_current"):
+            machine = await sync_to_async(Machine.current, thread_sensitive=True)()
+        with _perf_span("archivebox.MachineService.on_MachineEvent.prepare_config"):
+            old_config = dict(machine.config or {})
+            config = dict(old_config)
 
-        if event.config is not None:
-            binary_only = _strip_to_binary_keys(event.config)
-            config.update(_sanitize_machine_config(binary_only, lib_dir=lib_dir))
-        elif event.method == "update":
-            key = event.key.replace("config/", "", 1).strip()
-            if key and _is_binary_event_key(key):
-                config[key] = event.value
-        elif event.method == "unset":
-            key = event.key.replace("config/", "", 1).strip()
-            if key and _is_binary_event_key(key):
-                config.pop(key, None)
-        else:
+        with _perf_span("archivebox.MachineService.on_MachineEvent.merge_config"):
+            if event.config is not None:
+                binary_only = _strip_to_binary_keys(event.config)
+                config.update(binary_only)
+            elif event.method == "update":
+                key = event.key.replace("config/", "", 1).strip()
+                if key and _is_binary_event_key(key):
+                    config[key] = event.value
+            elif event.method == "unset":
+                key = event.key.replace("config/", "", 1).strip()
+                if key and _is_binary_event_key(key):
+                    config.pop(key, None)
+            else:
+                return
+
+        if config == old_config:
             return
-
-        machine.config = _sanitize_machine_config(config, lib_dir=lib_dir)
-        await machine.asave(update_fields=["config", "modified_at"])
+        machine.config = config
+        with _perf_span("archivebox.MachineService.on_MachineEvent.save"):
+            await machine.asave(update_fields=["config", "modified_at"])

archivebox/services/process_service.py

@@ -1,7 +1,12 @@
 from __future__ import annotations
 
 import asyncio
+import os
+import sys
+import time
+from contextlib import contextmanager
 from datetime import datetime
+from functools import wraps
 from typing import ClassVar
 
 from asgiref.sync import sync_to_async
@@ -12,6 +17,41 @@ from abx_dl.events import CrawlCleanupEvent, CrawlCompletedEvent, ProcessComplet
 from abx_dl.services.base import BaseService
 
 
+def _perf_trace(label):
+    def decorator(func):
+        if asyncio.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                    return await func(*args, **kwargs)
+                started_at = time.perf_counter()
+                try:
+                    return await func(*args, **kwargs)
+                finally:
+                    elapsed_ms = (time.perf_counter() - started_at) * 1000
+                    print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+            return async_wrapper
+
+        return func
+
+    return decorator
+
+
+@contextmanager
+def _perf_span(label: str):
+    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+        yield
+        return
+    started_at = time.perf_counter()
+    try:
+        yield
+    finally:
+        elapsed_ms = (time.perf_counter() - started_at) * 1000
+        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+
 def parse_event_datetime(value: str | None):
     if not value:
         return None
@@ -55,76 +95,85 @@ class ProcessService(BaseService):
             self._iface = await sync_to_async(current_network_interface_with_machine, thread_sensitive=True)()
         return self._iface
 
+    @_perf_trace("archivebox.ProcessService.on_ProcessStartedEvent__save_to_db")
     async def on_ProcessStartedEvent__save_to_db(self, event: ProcessStartedEvent) -> None:
         from archivebox.machine.models import Process
 
-        iface = await self.current_iface()
-        process_type = event.process_type or Process.TypeChoices.HOOK
-        worker_type = event.worker_type or ""
-        started_at = parse_event_datetime(event.start_ts)
+        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.current_iface"):
+            iface = await self.current_iface()
+        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.prepare_fields"):
+            process_type = event.process_type or Process.TypeChoices.HOOK
+            worker_type = event.worker_type or ""
+            started_at = parse_event_datetime(event.start_ts)
         if started_at is None:
             raise ValueError("ProcessStartedEvent.start_ts is required")
-        if event.pid:
-            process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
-        else:
-            process_query = Process.objects.filter(
-                process_type=process_type,
-                worker_type=worker_type,
-                pwd=event.output_dir,
-                started_at=started_at,
-            )
-        process = await process_query.order_by("-modified_at").afirst()
+        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.lookup"):
+            if event.pid:
+                process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
+            else:
+                process_query = Process.objects.filter(
+                    process_type=process_type,
+                    worker_type=worker_type,
+                    pwd=event.output_dir,
+                    started_at=started_at,
+                )
+            process = await process_query.order_by("-modified_at").afirst()
         if process is None:
-            process = await Process.objects.acreate(
-                machine=iface.machine,
-                iface=iface,
-                process_type=process_type,
-                worker_type=worker_type,
-                pwd=event.output_dir,
-                cmd=[event.hook_path, *event.hook_args],
-                env=event.env,
-                timeout=event.timeout,
-                pid=event.pid or None,
-                url=event.url or None,
-                started_at=started_at,
-                status=Process.StatusChoices.RUNNING,
-                retry_at=None,
-            )
+            with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.create"):
+                process = await Process.objects.acreate(
+                    machine=iface.machine,
+                    iface=iface,
+                    process_type=process_type,
+                    worker_type=worker_type,
+                    pwd=event.output_dir,
+                    cmd=[event.hook_path, *event.hook_args],
+                    env=event.env,
+                    timeout=event.timeout,
+                    pid=event.pid or None,
+                    url=event.url or None,
+                    started_at=started_at,
+                    status=Process.StatusChoices.RUNNING,
+                    retry_at=None,
+                )
         elif process.iface_id != iface.id or process.machine_id != iface.machine_id:
             process.iface = iface
             process.machine = iface.machine
-            await process.asave(update_fields=["iface", "machine", "modified_at"])
+            with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.update_iface"):
+                await process.asave(update_fields=["iface", "machine", "modified_at"])
 
-        process.pwd = event.output_dir
-        process.cmd = [event.hook_path, *event.hook_args]
-        process.env = event.env
-        process.timeout = event.timeout
-        process.pid = event.pid or None
-        process.url = event.url or process.url
-        process.process_type = process_type or process.process_type
-        process.worker_type = worker_type or process.worker_type
-        process.started_at = started_at
-        process.status = process.StatusChoices.RUNNING
-        process.retry_at = None
-        await sync_to_async(process.hydrate_binary_from_context, thread_sensitive=True)(
-            plugin_name=event.plugin_name,
-            hook_path=event.hook_path,
-        )
-        await Process.objects.filter(id=process.id).aupdate(
-            pwd=process.pwd,
-            cmd=process.cmd,
-            env=process.env,
-            timeout=process.timeout,
-            pid=process.pid,
-            url=process.url,
-            process_type=process.process_type,
-            worker_type=process.worker_type,
-            started_at=process.started_at,
-            status=process.status,
-            retry_at=process.retry_at,
-            binary_id=process.binary_id,
-            modified_at=timezone.now(),
-        )
+        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.assign_fields"):
+            process.pwd = event.output_dir
+            process.cmd = [event.hook_path, *event.hook_args]
+            process.env = event.env
+            process.timeout = event.timeout
+            process.pid = event.pid or None
+            process.url = event.url or process.url
+            process.process_type = process_type or process.process_type
+            process.worker_type = worker_type or process.worker_type
+            process.started_at = started_at
+            process.status = process.StatusChoices.RUNNING
+            process.retry_at = None
+        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.hydrate_binary"):
+            await sync_to_async(process.hydrate_binary_from_context, thread_sensitive=True)(
+                plugin_name=event.plugin_name,
+                hook_path=event.hook_path,
+            )
+        with _perf_span("archivebox.ProcessService.on_ProcessStartedEvent.update"):
+            await Process.objects.filter(id=process.id).aupdate(
+                pwd=process.pwd,
+                cmd=process.cmd,
+                env=process.env,
+                timeout=process.timeout,
+                pid=process.pid,
+                url=process.url,
+                process_type=process.process_type,
+                worker_type=process.worker_type,
+                started_at=process.started_at,
+                status=process.status,
+                retry_at=process.retry_at,
+                binary_id=process.binary_id,
+                modified_at=timezone.now(),
+            )
 
     async def _completed_worker_loop(self) -> None:
         while True:
@@ -153,63 +202,70 @@ class ProcessService(BaseService):
     async def on_CrawlCompletedEvent__flush_completed(self, event: CrawlCompletedEvent) -> None:
         await self.flush_completed()
 
+    @_perf_trace("archivebox.ProcessService._save_completed_process_to_db")
     async def _save_completed_process_to_db(self, event: ProcessCompletedEvent) -> None:
         from archivebox.machine.models import Process
 
-        iface = await self.current_iface()
-        process_type = event.process_type or Process.TypeChoices.HOOK
-        worker_type = event.worker_type or ""
-        started_at = parse_event_datetime(event.start_ts)
+        with _perf_span("archivebox.ProcessService._save_completed.current_iface"):
+            iface = await self.current_iface()
+        with _perf_span("archivebox.ProcessService._save_completed.prepare_fields"):
+            process_type = event.process_type or Process.TypeChoices.HOOK
+            worker_type = event.worker_type or ""
+            started_at = parse_event_datetime(event.start_ts)
         if started_at is None:
             raise ValueError("ProcessCompletedEvent.start_ts is required")
-        if event.pid:
-            process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
-        else:
-            process_query = Process.objects.filter(
-                process_type=process_type,
-                worker_type=worker_type,
-                pwd=event.output_dir,
-                started_at=started_at,
-            )
-        process = await process_query.order_by("-modified_at").afirst()
-        if process is None:
-            await Process.objects.acreate(
-                machine=iface.machine,
-                iface=iface,
-                process_type=process_type,
-                worker_type=worker_type,
-                pwd=event.output_dir,
-                cmd=[event.hook_path, *event.hook_args],
-                env=event.env,
-                timeout=event.timeout,
-                pid=event.pid or None,
-                url=event.url or None,
-                started_at=started_at,
-                status=Process.StatusChoices.RUNNING,
-                retry_at=None,
-            )
+        with _perf_span("archivebox.ProcessService._save_completed.lookup"):
+            if event.pid:
+                process_query = Process.objects.filter(pid=event.pid, started_at=started_at)
+            else:
+                process_query = Process.objects.filter(
+                    process_type=process_type,
+                    worker_type=worker_type,
+                    pwd=event.output_dir,
+                    started_at=started_at,
+                )
             process = await process_query.order_by("-modified_at").afirst()
+        if process is None:
+            with _perf_span("archivebox.ProcessService._save_completed.create_missing"):
+                await Process.objects.acreate(
+                    machine=iface.machine,
+                    iface=iface,
+                    process_type=process_type,
+                    worker_type=worker_type,
+                    pwd=event.output_dir,
+                    cmd=[event.hook_path, *event.hook_args],
+                    env=event.env,
+                    timeout=event.timeout,
+                    pid=event.pid or None,
+                    url=event.url or None,
+                    started_at=started_at,
+                    status=Process.StatusChoices.RUNNING,
+                    retry_at=None,
+                )
+                process = await process_query.order_by("-modified_at").afirst()
             if process is None:
                 return
 
-        missing_cmd = not process.cmd
-        updates = {
-            "machine_id": iface.machine_id,
-            "iface_id": iface.id,
-            "pwd": event.output_dir,
-            "pid": event.pid or process.pid,
-            "url": event.url or process.url,
-            "process_type": process_type or process.process_type,
-            "worker_type": worker_type or process.worker_type,
-            "started_at": started_at,
-            "ended_at": parse_event_datetime(event.end_ts) or timezone.now(),
-            "stdout": event.stdout,
-            "stderr": event.stderr,
-            "exit_code": event.exit_code,
-            "status": Process.StatusChoices.EXITED,
-            "retry_at": None,
-            "modified_at": timezone.now(),
-        }
-        if missing_cmd:
-            updates["cmd"] = [event.hook_path, *event.hook_args]
-        await Process.objects.filter(id=process.id).aupdate(**updates)
+        with _perf_span("archivebox.ProcessService._save_completed.prepare_updates"):
+            missing_cmd = not process.cmd
+            updates = {
+                "machine_id": iface.machine_id,
+                "iface_id": iface.id,
+                "pwd": event.output_dir,
+                "pid": event.pid or process.pid,
+                "url": event.url or process.url,
+                "process_type": process_type or process.process_type,
+                "worker_type": worker_type or process.worker_type,
+                "started_at": started_at,
+                "ended_at": parse_event_datetime(event.end_ts) or timezone.now(),
+                "stdout": event.stdout,
+                "stderr": event.stderr,
+                "exit_code": event.exit_code,
+                "status": Process.StatusChoices.EXITED,
+                "retry_at": None,
+                "modified_at": timezone.now(),
+            }
+            if missing_cmd:
+                updates["cmd"] = [event.hook_path, *event.hook_args]
+        with _perf_span("archivebox.ProcessService._save_completed.update"):
+            await Process.objects.filter(id=process.id).aupdate(**updates)

archivebox/services/snapshot_service.py

@@ -1,6 +1,11 @@
 from __future__ import annotations
 
+import asyncio
 import sys
+import os
+import time
+from contextlib import contextmanager
+from functools import wraps
 
 from asgiref.sync import sync_to_async
 from django.utils import timezone
@@ -11,43 +16,105 @@ from abx_dl.limits import CrawlLimitState
 from abx_dl.services.base import BaseService
 
 
-def finalize_completed_snapshot(snapshot_id: str) -> None:
+def _perf_trace(label):
+    def decorator(func):
+        if asyncio.iscoroutinefunction(func):
+
+            @wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                    return await func(*args, **kwargs)
+                started_at = time.perf_counter()
+                try:
+                    return await func(*args, **kwargs)
+                finally:
+                    elapsed_ms = (time.perf_counter() - started_at) * 1000
+                    print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+            return async_wrapper
+
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+                return func(*args, **kwargs)
+            started_at = time.perf_counter()
+            try:
+                return func(*args, **kwargs)
+            finally:
+                elapsed_ms = (time.perf_counter() - started_at) * 1000
+                print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+        return sync_wrapper
+
+    return decorator
+
+
+@contextmanager
+def _perf_span(label: str):
+    if os.environ.get("ARCHIVEBOX_PERF_TRACE") != "1":
+        yield
+        return
+    started_at = time.perf_counter()
+    try:
+        yield
+    finally:
+        elapsed_ms = (time.perf_counter() - started_at) * 1000
+        print(f"PERF_TRACE label={label} ms={elapsed_ms:.3f}", file=sys.stderr, flush=True)
+
+
+@_perf_trace("archivebox.SnapshotService.finalize_completed_snapshot")
+def finalize_completed_snapshot(
+    snapshot_id: str,
+    *,
+    output_dir=None,
+    crawl_limit_stop_reason: str | None = None,
+) -> None:
     from archivebox.core.models import Snapshot
 
-    snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
+    with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.snapshot_lookup"):
+        snapshot = Snapshot.objects.select_related("crawl", "crawl__created_by").filter(id=snapshot_id).first()
     if snapshot is None:
         return
 
     if snapshot.downloaded_at is None:
-        snapshot.downloaded_at = timezone.now()
-        snapshot.save(update_fields=["downloaded_at", "modified_at"])
+        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.mark_downloaded"):
+            snapshot.downloaded_at = timezone.now()
+            snapshot.save(update_fields=["downloaded_at", "modified_at"])
 
-    stop_reason = _crawl_limit_stop_reason(snapshot.crawl)
+    with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.stop_reason"):
+        stop_reason = crawl_limit_stop_reason if crawl_limit_stop_reason is not None else _crawl_limit_stop_reason(snapshot.crawl)
     if snapshot.crawl_id and stop_reason in ("crawl_max_size", "crawl_timeout"):
-        Snapshot.objects.filter(
-            crawl_id=snapshot.crawl_id,
-            status=Snapshot.StatusChoices.QUEUED,
-        ).exclude(id=snapshot.id).update(
-            status=Snapshot.StatusChoices.SEALED,
-            retry_at=None,
-            modified_at=timezone.now(),
-        )
+        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.seal_due_to_limits"):
+            Snapshot.objects.filter(
+                crawl_id=snapshot.crawl_id,
+                status=Snapshot.StatusChoices.QUEUED,
+            ).exclude(id=snapshot.id).update(
+                status=Snapshot.StatusChoices.SEALED,
+                retry_at=None,
+                modified_at=timezone.now(),
+            )
 
     if snapshot.status == Snapshot.StatusChoices.QUEUED:
-        snapshot.sm.tick()
-        snapshot.refresh_from_db()
+        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.tick_queued"):
+            snapshot.sm.tick()
+            snapshot.refresh_from_db()
     if snapshot.status == Snapshot.StatusChoices.STARTED and snapshot.is_finished_processing():
-        snapshot.sm.seal()
-        snapshot.refresh_from_db()
+        with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.seal_finished"):
+            snapshot.sm.seal()
+            snapshot.refresh_from_db()
 
-    snapshot.write_index_jsonl()
+    with _perf_span("archivebox.SnapshotService.finalize_completed_snapshot.write_index_jsonl"):
+        snapshot.write_index_jsonl(output_dir=output_dir)
 
 
 def _crawl_limit_stop_reason(crawl) -> str:
     from archivebox.config.common import get_config
 
-    config = get_config(crawl=crawl, include_machine=False)
-    config["CRAWL_DIR"] = str(crawl.output_dir)
+    config_model = get_config(crawl=crawl)
+    config = config_model.for_crawl_runtime(
+        crawl=crawl,
+        persona=crawl.resolve_persona(),
+    )
     return CrawlLimitState.from_config(config).get_stop_reason()
 
 
@@ -62,36 +129,43 @@ class SnapshotService(BaseService):
         self.bus.on(SnapshotEvent, self.on_SnapshotEvent)
         self.bus.on(SnapshotCompletedEvent, self.on_SnapshotCompletedEvent)
 
+    @_perf_trace("archivebox.SnapshotService.on_SnapshotEvent")
     async def on_SnapshotEvent(self, event: SnapshotEvent) -> None:
         from archivebox.core.models import Snapshot
 
-        snapshot = await Snapshot.objects.filter(id=event.snapshot_id, crawl_id=self.crawl_id).afirst()
+        with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.snapshot_lookup"):
+            snapshot = await Snapshot.objects.filter(id=event.snapshot_id, crawl_id=self.crawl_id).afirst()
 
         if snapshot is not None:
             if snapshot.is_paused:
                 return
             if snapshot.status == Snapshot.StatusChoices.QUEUED:
                 try:
-                    await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
+                    with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.tick_queued"):
+                        await sync_to_async(snapshot.sm.tick, thread_sensitive=True)()
                 except ValidationError as err:
                     if "ArchiveBox cannot archive its own admin, web, api, or snapshot URLs." not in str(err):
                         raise
-                    await Snapshot.objects.filter(id=snapshot.id).aupdate(
-                        status=Snapshot.StatusChoices.SEALED,
-                        retry_at=None,
-                        modified_at=timezone.now(),
-                    )
+                    with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.seal_invalid_internal_url"):
+                        await Snapshot.objects.filter(id=snapshot.id).aupdate(
+                            status=Snapshot.StatusChoices.SEALED,
+                            retry_at=None,
+                            modified_at=timezone.now(),
+                        )
                     rprint(
                         f"[red][X] Refusing to archive ArchiveBox internal URL for security: {snapshot.url}[/red]",
                         file=sys.stderr,
                     )
                     return
-                await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
+                with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.refresh_after_tick"):
+                    await sync_to_async(snapshot.refresh_from_db, thread_sensitive=True)()
             elif snapshot.status != Snapshot.StatusChoices.STARTED:
                 return
             if snapshot.status != Snapshot.StatusChoices.STARTED:
                 return
-            await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
+            with _perf_span("archivebox.SnapshotService.on_SnapshotEvent.ensure_crawl_symlink"):
+                await sync_to_async(snapshot.ensure_crawl_symlink, thread_sensitive=True)()
 
+    @_perf_trace("archivebox.SnapshotService.on_SnapshotCompletedEvent")
     async def on_SnapshotCompletedEvent(self, event: SnapshotCompletedEvent) -> None:
         await sync_to_async(finalize_completed_snapshot, thread_sensitive=True)(event.snapshot_id)

archivebox/templates/core/add.html

@@ -304,7 +304,7 @@
                                 <div class="error">{{ form.config.errors }}</div>
                             {% endif %}
                             <div class="help-text">
-                                Override any config option for this crawl (e.g., TIMEOUT, USER_AGENT, CHROME_BINARY, etc.). <code>URL_ALLOWLIST</code>, <code>URL_DENYLIST</code>, and <code>PLUGINS</code> are updated automatically from the fields above.
+                                Override crawl-scoped config options (e.g., TIMEOUT, USER_AGENT, URL_ALLOWLIST, URL_DENYLIST, PLUGINS). <code>*_BINARY</code> paths are managed on Persona or Machine config.
                             </div>
                         </div>
                     </details>

archivebox/tests/conftest.py

@@ -25,19 +25,10 @@ pytest_plugins = ["archivebox.tests.fixtures"]
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
-SESSION_DATA_DIR = Path(
-    os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
-).resolve()
-# Force ArchiveBox imports to see a temp DATA_DIR during test collection.
-os.environ["ARCHIVEBOX_PYTEST_SESSION_DATA_DIR"] = str(SESSION_DATA_DIR)
-os.environ["DATA_DIR"] = str(SESSION_DATA_DIR)
+SESSION_DATA_DIR = Path(tempfile.mkdtemp(prefix="archivebox-pytest-session-")).resolve()
 (SESSION_DATA_DIR / "tests").mkdir(parents=True, exist_ok=True)
 os.chdir(SESSION_DATA_DIR)
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "archivebox.core.settings")
-os.environ.pop("ARCHIVE_DIR", None)
-os.environ.pop("USERS_DIR", None)
-os.environ.pop("CRAWL_DIR", None)
-os.environ.pop("SNAP_DIR", None)
 
 
 def _is_repo_path(path: Path) -> bool:
@@ -56,7 +47,7 @@ def _assert_safe_runtime_paths(*, cwd: Path | None = None, env: dict[str, str] |
     if cwd is not None:
         _assert_not_repo_path(cwd, label="cwd")
 
-    for key in ("DATA_DIR", "ARCHIVE_DIR", "USERS_DIR", "CRAWL_DIR", "SNAP_DIR"):
+    for key in ("CRAWL_DIR", "SNAP_DIR"):
         value = (env or {}).get(key)
         if value:
             _assert_not_repo_path(Path(value), label=key)
@@ -89,9 +80,8 @@ def run_archivebox_cmd(
     """
     cmd = [sys.executable, "-m", "archivebox"] + args
 
-    _assert_not_repo_path(data_dir, label="DATA_DIR")
+    _assert_not_repo_path(data_dir, label="cwd")
     base_env = os.environ.copy()
-    base_env["DATA_DIR"] = str(data_dir)
     base_env["USE_COLOR"] = "False"
     base_env["SHOW_PROGRESS"] = "False"
     # Disable slow extractors for faster tests
@@ -143,9 +133,8 @@ def isolate_test_runtime(tmp_path, monkeypatch):
     contract is that every test starts in its own temp directory and any
     in-process ``os.environ`` edits are rolled back afterwards.
 
-    Each in-process test gets an explicit temp ``DATA_DIR`` so ArchiveBox code
-    never falls back to the repo cwd. Subprocess helpers that intentionally test
-    cwd-based behavior remove ``DATA_DIR`` for the child process themselves.
+    ArchiveBox derives DATA_DIR from cwd, so subprocess helpers pass the target
+    collection as cwd instead of using DATA_DIR as an override.
     """
     _assert_not_repo_path(tmp_path, label="tmp_path")
     original_cwd = Path.cwd()
@@ -176,11 +165,6 @@ def isolate_test_runtime(tmp_path, monkeypatch):
 
     monkeypatch.setattr(os, "chdir", guarded_chdir)
     monkeypatch.setattr(subprocess, "Popen", guarded_popen)
-    os.environ["DATA_DIR"] = str(tmp_path)
-    os.environ.pop("ARCHIVE_DIR", None)
-    os.environ.pop("USERS_DIR", None)
-    os.environ.pop("CRAWL_DIR", None)
-    os.environ.pop("SNAP_DIR", None)
     reset_machine_model_caches()
     try:
         _assert_safe_runtime_paths(cwd=Path.cwd(), env=os.environ)
@@ -247,7 +231,7 @@ def initialized_archive(isolated_data_dir):
 
 
 @pytest.fixture
-def archivebox_daemon_server(tmp_path, process, unused_tcp_port_factory):
+def archivebox_daemon_server(tmp_path, process, free_tcp_port_factory):
     """
     Start a real daemonized ArchiveBox server in this test's DATA_DIR and
     always stop its supervisord before the test exits.
@@ -262,11 +246,11 @@ def archivebox_daemon_server(tmp_path, process, unused_tcp_port_factory):
                 "USE_COLOR": "False",
                 "SHOW_PROGRESS": "False",
                 "SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
-                "SEARCH_BACKEND_SONIC_PORT": str(unused_tcp_port_factory()),
+                "SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
                 **{key: str(value) for key, value in env_overrides.items()},
             },
         )
-        port = unused_tcp_port_factory()
+        port = free_tcp_port_factory()
         result = subprocess.run(
             [sys.executable, "-m", "archivebox", "server", "--daemonize", f"127.0.0.1:{port}"],
             cwd=tmp_path,
@@ -330,11 +314,6 @@ def run_archivebox_cmd_cwd(
 
     _assert_not_repo_path(cwd, label="cwd")
     base_env = os.environ.copy()
-    base_env.pop("DATA_DIR", None)
-    base_env.pop("ARCHIVE_DIR", None)
-    base_env.pop("USERS_DIR", None)
-    base_env.pop("CRAWL_DIR", None)
-    base_env.pop("SNAP_DIR", None)
     base_env["USE_COLOR"] = "False"
     base_env["SHOW_PROGRESS"] = "False"
 
@@ -437,11 +416,6 @@ def run_python_cwd(
 ) -> tuple[str, str, int]:
     _assert_not_repo_path(cwd, label="cwd")
     base_env = os.environ.copy()
-    base_env.pop("DATA_DIR", None)
-    base_env.pop("ARCHIVE_DIR", None)
-    base_env.pop("USERS_DIR", None)
-    base_env.pop("CRAWL_DIR", None)
-    base_env.pop("SNAP_DIR", None)
     _assert_safe_runtime_paths(cwd=cwd, env=base_env)
     result = subprocess.run(
         [sys.executable, "-"],
@@ -473,7 +447,6 @@ def init_archive(cwd: Path) -> None:
 
 def build_test_env(port: int, **extra: str) -> dict[str, str]:
     env = os.environ.copy()
-    env.pop("DATA_DIR", None)
     env.update(
         {
             "PLUGINS": "wget",
@@ -874,7 +847,7 @@ def real_archive_with_example(tmp_path_factory, request):
     Uses cwd for DATA_DIR.
     """
     tmp_path = tmp_path_factory.mktemp("archivebox_data")
-    if getattr(request, "cls", None) is not None:
+    if request.cls is not None:
         request.cls.data_dir = tmp_path
 
     stdout, stderr, returncode = run_archivebox_cmd_cwd(

archivebox/tests/migrations_helpers.py

@@ -1023,7 +1023,6 @@ def seed_0_8_data(db_path: Path) -> dict[str, list[dict]]:
 def run_archivebox(data_dir: Path, args: list, timeout: int = 60, env: dict | None = None) -> subprocess.CompletedProcess:
     """Run archivebox command in subprocess with given data directory."""
     base_env = os.environ.copy()
-    base_env["DATA_DIR"] = str(data_dir)
     base_env["USE_COLOR"] = "False"
     base_env["SHOW_PROGRESS"] = "False"
     # Disable ALL extractors for faster tests (can be overridden by env parameter)

archivebox/tests/test_archive_result_service.py

@@ -22,15 +22,19 @@ def _create_snapshot():
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
 
-    crawl = Crawl.objects.create(
+    crawl = Crawl(
         urls="https://example.com",
         created_by_id=get_or_create_system_user_pk(),
     )
-    return Snapshot.objects.create(
+    crawl.save()
+
+    snapshot = Snapshot(
         url="https://example.com",
         crawl=crawl,
         status=Snapshot.StatusChoices.STARTED,
     )
+    snapshot.save()
+    return snapshot
 
 
 def test_process_completed_projects_inline_archiveresult():
@@ -236,6 +240,42 @@ def test_snapshot_resolved_title_ignores_failed_title_output_str():
     _cleanup_machine_process_rows()
 
 
+def test_snapshot_title_ignores_noresults_title_output_str():
+    from archivebox.core.models import ArchiveResult
+    from archivebox.services.archive_result_service import ArchiveResultService
+    import asyncio
+
+    snapshot = _create_snapshot()
+    plugin_dir = Path(snapshot.output_dir) / "title"
+    plugin_dir.mkdir(parents=True, exist_ok=True)
+
+    bus = create_bus(name="test_noresults_title_does_not_update_snapshot")
+    service = ArchiveResultService(bus)
+
+    event = ArchiveResultEvent(
+        snapshot_id=str(snapshot.id),
+        plugin="title",
+        hook_name="on_Snapshot__54_title.js",
+        status="noresults",
+        output_str="TimeoutError: Navigation timeout of 54172 ms exceeded",
+        start_ts="2026-03-22T12:00:00+00:00",
+        end_ts="2026-03-22T12:00:01+00:00",
+    )
+
+    async def emit_event() -> None:
+        await service.on_ArchiveResultEvent__save_to_db(event)
+
+    asyncio.run(emit_event())
+
+    result = ArchiveResult.objects.get(snapshot=snapshot, plugin="title", hook_name="on_Snapshot__54_title.js")
+    assert result.status == ArchiveResult.StatusChoices.NORESULTS
+    assert result.output_str == "TimeoutError: Navigation timeout of 54172 ms exceeded"
+    snapshot.refresh_from_db()
+    assert snapshot.title in (None, "")
+    assert snapshot.resolved_title == ""
+    _cleanup_machine_process_rows()
+
+
 def test_snapshot_save_normalizes_url_title_to_none():
     from archivebox.core.models import Snapshot
 

archivebox/tests/test_auth_ldap.py

@@ -117,14 +117,14 @@ class TestLDAPAuthBackend:
         """Test that ArchiveBoxLDAPBackend class is defined."""
         from archivebox.ldap.auth import ArchiveBoxLDAPBackend
 
-        assert hasattr(ArchiveBoxLDAPBackend, "authenticate_ldap_user")
+        assert ArchiveBoxLDAPBackend.authenticate_ldap_user is not None
 
     def test_ldap_backend_inherits_correctly(self):
         """Test that ArchiveBoxLDAPBackend has correct inheritance."""
         from archivebox.ldap.auth import ArchiveBoxLDAPBackend
 
         # Should have authenticate_ldap_user method (from base or overridden)
-        assert callable(getattr(ArchiveBoxLDAPBackend, "authenticate_ldap_user", None))
+        assert callable(ArchiveBoxLDAPBackend.authenticate_ldap_user)
 
 
 class TestArchiveBoxWithLDAP:

archivebox/tests/test_cli_add.py

@@ -269,7 +269,7 @@ def test_add_records_selected_persona_on_crawl(tmp_path, process, disable_extrac
         crawl = Crawl.objects.get()
 
     assert crawl.persona_id
-    assert crawl.config["ACTIVE_PERSONA"] == "Default"
+    assert "ACTIVE_PERSONA" not in crawl.config
     assert (tmp_path / "personas" / "Default" / "chrome_profile").is_dir()
 
 

archivebox/tests/test_cli_init.py

@@ -46,21 +46,15 @@ def test_init_creates_archive_directory(tmp_path):
     assert archive_dir.is_dir()
 
 
-def test_init_respects_configured_archive_and_users_dirs(tmp_path):
-    """Test that init creates configured archive/users storage roots."""
+def test_init_uses_cwd_archive_and_users_dirs(tmp_path):
+    """Test that init creates archive/users storage roots under cwd."""
     os.chdir(tmp_path)
-    archive_dir = tmp_path / "mounted_archive"
-    users_dir = archive_dir / "custom_users"
-    env = os.environ.copy()
-    env["ARCHIVE_DIR"] = str(archive_dir)
-    env["USERS_DIR"] = str(users_dir)
 
-    result = subprocess.run(["archivebox", "init"], env=env, capture_output=True)
+    result = subprocess.run(["archivebox", "init"], capture_output=True)
 
     assert result.returncode == 0
-    assert archive_dir.is_dir()
-    assert users_dir.is_dir()
-    assert not (tmp_path / "archive").exists()
+    assert (tmp_path / "archive").is_dir()
+    assert (tmp_path / "archive" / "users").is_dir()
 
 
 def test_init_creates_sources_directory(tmp_path):

archivebox/tests/test_cli_run.py

@@ -402,7 +402,7 @@ class TestRunDaemonMode:
 
         stdout = proc.stdout.read()
         stderr = proc.stderr.read()
-        assert proc.returncode == 0, stdout + stderr
+        assert proc.returncode == 143, stdout + stderr
         assert "No records to process" not in stderr
 
     def test_run_daemon_takeover_has_single_active_runner_gate(self, initialized_archive, db):
@@ -1388,6 +1388,41 @@ class TestRecoverOrchestratorState:
 
 @pytest.mark.django_db
 class TestRunDueCrawlState:
+    def test_idle_maintenance_repairs_archive_result_delete_at(self):
+        from archivebox.base_models.models import get_or_create_system_user_pk
+        from archivebox.crawls.models import Crawl
+        from archivebox.core.models import ArchiveResult, Snapshot
+        from archivebox.services.runner import run_pending_crawls
+
+        crawl = Crawl.objects.create(
+            urls="https://example.com/retention-repair",
+            created_by_id=get_or_create_system_user_pk(),
+            status=Crawl.StatusChoices.SEALED,
+            retry_at=None,
+            config={"DELETE_AFTER": "2h"},
+        )
+        snapshot = Snapshot.objects.create(
+            url="https://example.com/retention-repair",
+            crawl=crawl,
+            status=Snapshot.StatusChoices.SEALED,
+            retry_at=None,
+        )
+        result = ArchiveResult.objects.create(
+            snapshot=snapshot,
+            plugin="search_backend_sqlite",
+            hook_name="on_Snapshot__90_index_sqlite.py",
+            status=ArchiveResult.StatusChoices.SUCCEEDED,
+        )
+
+        # ArchiveResult saves are the plugin-event hot path. They intentionally
+        # do not resolve parent Snapshot/Crawl config on every write; the real
+        # runner's idle maintenance pass owns missing delete_at repair.
+        assert result.delete_at is None
+        assert run_pending_crawls(daemon=False, maintenance_only=True) == 0
+
+        result.refresh_from_db()
+        assert result.delete_at is not None
+
     def test_maintenance_only_runner_does_not_start_regular_queued_crawls(self):
         from django.utils import timezone
 

archivebox/tests/test_cli_server.py

@@ -11,6 +11,7 @@ import signal
 import socket
 import subprocess
 import sys
+import time
 from datetime import datetime
 from types import SimpleNamespace
 
@@ -128,6 +129,7 @@ def test_runner_worker_uses_current_interpreter():
     from archivebox.workers.supervisord_util import RUNNER_WORKER
 
     assert RUNNER_WORKER["command"] == f"{sys.executable} -m archivebox run --daemon"
+    assert 'ARCHIVEBOX_RUNNER_DAEMON="1"' in RUNNER_WORKER["environment"]
 
 
 def test_daphne_worker_uses_default_application_close_timeout():
@@ -175,13 +177,105 @@ def test_server_daemon_starts_real_plugin_owned_sonic_worker(archivebox_daemon_s
     assert "sonic" in state["worker_sonic"]["name"]
 
 
-def test_sonic_worker_is_disabled_when_sonic_disabled_and_engine_not_sonic(tmp_path):
+def test_server_daemon_restarts_runner_killed_by_signal(archivebox_daemon_server):
+    server = archivebox_daemon_server(
+        SEARCH_BACKEND_ENGINE="sqlite",
+    )
+    state = server.wait_for_workers(("worker_daphne", "worker_runner"))
+    old_runner_pid = state["worker_runner"]["pid"]
+
+    os.kill(old_runner_pid, signal.SIGTERM)
+
+    deadline = time.time() + 30
+    while time.time() < deadline:
+        state = server.worker_state()
+        runner = state.get("worker_runner", {})
+        if runner.get("statename") == "RUNNING" and runner.get("pid") and runner.get("pid") != old_runner_pid:
+            break
+        time.sleep(0.5)
+    else:
+        raise AssertionError(f"worker_runner did not restart after SIGTERM: {state}")
+
+    assert state["worker_daphne"]["statename"] == "RUNNING", state
+
+
+def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_runtime(archivebox_daemon_server):
+    server = archivebox_daemon_server(SEARCH_BACKEND_ENGINE="ripgrep")
+    server.wait_for_workers(("worker_daphne", "worker_runner"))
+
+    setup_result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import django;"
+                "django.setup();"
+                "from archivebox.base_models.models import get_or_create_system_user_pk;"
+                "from archivebox.crawls.models import Crawl;"
+                "from archivebox.core.models import Snapshot;"
+                "from archivebox.machine.models import Machine;"
+                "machine = Machine.current(refresh=True);"
+                "machine.config = {**dict(machine.config or {}), 'SEARCH_BACKEND_ENGINE': 'sqlite'};"
+                "machine.save(update_fields=['config', 'modified_at']);"
+                "crawl = Crawl.objects.create("
+                "urls='https://example.com/live-machine-search-config',"
+                "created_by_id=get_or_create_system_user_pk(),"
+                "config={},"
+                ");"
+                "snapshot = Snapshot.objects.create("
+                "url='https://example.com/live-machine-search-config',"
+                "crawl=crawl,"
+                ");"
+                "print(snapshot.id)"
+            ),
+        ],
+        cwd=server.data_dir,
+        env=server.env,
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    assert setup_result.returncode == 0, setup_result.stderr or setup_result.stdout
+    snapshot_id = setup_result.stdout.strip().splitlines()[-1]
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import django,json;"
+                "django.setup();"
+                "from archivebox.core.models import Snapshot;"
+                "from archivebox.config.common import get_config;"
+                f"snapshot = Snapshot.objects.select_related('crawl').get(id='{snapshot_id}');"
+                "runtime = get_config(snapshot=snapshot).for_crawl_runtime("
+                "crawl=snapshot.crawl,"
+                "snapshot=snapshot,"
+                "extra_context={'snapshot_id': str(snapshot.id)},"
+                ");"
+                "print(json.dumps({"
+                "'sqlite_enabled': runtime.get('SEARCH_BACKEND_SQLITE_ENABLED'),"
+                "'engine_in_runtime': 'SEARCH_BACKEND_ENGINE' in runtime,"
+                "}))"
+            ),
+        ],
+        cwd=server.data_dir,
+        env=server.env,
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    assert result.returncode == 0, result.stderr or result.stdout
+    resolved = json.loads(result.stdout.strip().splitlines()[-1])
+    assert resolved == {"sqlite_enabled": True, "engine_in_runtime": False}
+
+
+def test_sonic_worker_is_disabled_when_sonic_disabled(tmp_path):
     from archivebox.workers.supervisord_util import get_sonic_supervisord_worker_from_plugin
 
     worker = get_sonic_supervisord_worker_from_plugin(
         SimpleNamespace(
             DATA_DIR=str(tmp_path),
-            SEARCH_BACKEND_ENGINE="ripgrep",
             SEARCH_BACKEND_SONIC_ENABLED=False,
             SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
             SEARCH_BACKEND_SONIC_PORT=_free_port(),
@@ -210,7 +304,7 @@ def test_sonic_daemon_event_handler_accepts_real_running_worker(archivebox_daemo
     daemon_event = prepare_sonic_daemon(
         SimpleNamespace(
             DATA_DIR=str(server.data_dir),
-            SEARCH_BACKEND_ENGINE="sonic",
+            SEARCH_BACKEND_SONIC_ENABLED=True,
             SEARCH_BACKEND_SONIC_HOST_NAME="127.0.0.1",
             SEARCH_BACKEND_SONIC_PORT=sonic_port,
             SEARCH_BACKEND_SONIC_PASSWORD="SecretPassword",

archivebox/tests/test_crawl_runner.py

@@ -105,30 +105,82 @@ def test_enqueue_discovered_snapshots_refreshes_crawl_limits(tmp_path):
 
 
 @pytest.mark.django_db(transaction=True)
-def test_snapshot_payload_uses_crawl_persona_runtime_dirs():
+def test_snapshot_payload_uses_crawl_chrome_dirs_by_default():
     from archivebox.base_models.models import get_or_create_system_user_pk
     from archivebox.crawls.models import Crawl
     from archivebox.core.models import Snapshot
     from archivebox.personas.models import Persona
     from archivebox.services.runner import CrawlRunner
 
-    persona = Persona.objects.create(name="RuntimePersona")
-    crawl = Crawl.objects.create(
+    persona = Persona(name="RuntimePersona")
+    persona.save()
+    crawl = Crawl(
         urls="https://example.com",
         persona_id=persona.id,
         created_by_id=get_or_create_system_user_pk(),
     )
-    snapshot = Snapshot.objects.create(url="https://example.com", crawl=crawl)
+    crawl.save()
+    snapshot = Snapshot(url="https://example.com", crawl=crawl)
+    snapshot.save()
+    other_snapshot = Snapshot(url="https://example.org", crawl=crawl)
+    other_snapshot.save()
 
     runner = CrawlRunner(crawl)
     runner.load_run_state()
+    crawl_downloads_sentinel = persona.runtime_downloads_dir_for_crawl(crawl) / "keep.txt"
+    crawl_downloads_sentinel.write_text("keep")
     payload = runner.load_snapshot_payload(str(snapshot.id))
+    other_payload = runner.load_snapshot_payload(str(other_snapshot.id))
     config = payload["config"]
+    other_config = other_payload["config"]
 
     assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(crawl.output_dir)
     assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(crawl.output_dir)
     assert Path(config["CHROME_USER_DATA_DIR"]).name == "chrome_profile"
     assert Path(config["CHROME_DOWNLOADS_DIR"]).name == "chrome_downloads"
+    assert Path(config["CHROME_USER_DATA_DIR"]) == Path(other_config["CHROME_USER_DATA_DIR"])
+    assert Path(config["CHROME_DOWNLOADS_DIR"]) == Path(other_config["CHROME_DOWNLOADS_DIR"])
+    assert crawl_downloads_sentinel.read_text() == "keep"
+    assert config["ACTIVE_PERSONA"] == "RuntimePersona"
+    assert Path(config["CRAWL_DIR"]) == crawl.output_dir
+    assert Path(config["SNAP_DIR"]) == snapshot.output_dir
+
+
+@pytest.mark.django_db(transaction=True)
+def test_snapshot_payload_uses_snapshot_chrome_dirs_when_snapshot_isolated():
+    from archivebox.base_models.models import get_or_create_system_user_pk
+    from archivebox.crawls.models import Crawl
+    from archivebox.core.models import Snapshot
+    from archivebox.personas.models import Persona
+    from archivebox.services.runner import CrawlRunner
+
+    persona = Persona(name="SnapshotRuntimePersona")
+    persona.save()
+    crawl = Crawl(
+        urls="https://example.com\nhttps://example.org",
+        persona_id=persona.id,
+        created_by_id=get_or_create_system_user_pk(),
+        config={"CHROME_ISOLATION": "snapshot"},
+    )
+    crawl.save()
+    snapshot = Snapshot(url="https://example.com", crawl=crawl)
+    snapshot.save()
+    other_snapshot = Snapshot(url="https://example.org", crawl=crawl)
+    other_snapshot.save()
+
+    runner = CrawlRunner(crawl)
+    runner.load_run_state()
+    payload = runner.load_snapshot_payload(str(snapshot.id))
+    other_payload = runner.load_snapshot_payload(str(other_snapshot.id))
+    config = payload["config"]
+    other_config = other_payload["config"]
+
+    assert Path(config["CHROME_USER_DATA_DIR"]).is_relative_to(snapshot.output_dir)
+    assert Path(config["CHROME_DOWNLOADS_DIR"]).is_relative_to(snapshot.output_dir)
+    assert Path(other_config["CHROME_USER_DATA_DIR"]).is_relative_to(other_snapshot.output_dir)
+    assert Path(other_config["CHROME_DOWNLOADS_DIR"]).is_relative_to(other_snapshot.output_dir)
+    assert Path(config["CHROME_USER_DATA_DIR"]) != Path(other_config["CHROME_USER_DATA_DIR"])
+    assert Path(config["CHROME_DOWNLOADS_DIR"]) != Path(other_config["CHROME_DOWNLOADS_DIR"])
     assert Path(config["CRAWL_DIR"]) == crawl.output_dir
     assert Path(config["SNAP_DIR"]) == snapshot.output_dir
 
@@ -163,6 +215,18 @@ def test_ensure_background_runner_skips_with_real_running_orchestrator_record():
     assert process.status == Process.StatusChoices.RUNNING
 
 
+@pytest.mark.django_db(transaction=True)
+def test_ensure_background_runner_does_not_spawn_runner_without_supervisord():
+    from archivebox.services.runner import ensure_background_runner
+    from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_existing_supervisord_process
+
+    stop_existing_supervisord_process()
+    assert get_existing_supervisord_process(quiet=True) is None
+
+    assert ensure_background_runner(allow_under_pytest=True) is False
+    assert get_existing_supervisord_process(quiet=True) is None
+
+
 def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
     from abx_dl.events import CrawlEvent, MachineEvent
     from abx_dl.orchestrator import create_bus
@@ -174,7 +238,7 @@ def test_runner_task_context_clears_inherited_abxbus_handler_context(tmp_path):
 
     async def emit_from_runner_task():
         observations.append(("in_handler_context", in_handler_context()))
-        machine_event = bus.emit(MachineEvent(config={"ABX_RUNTIME": "archivebox"}, config_type="user"))
+        machine_event = bus.emit(MachineEvent(config={"TIMEOUT": "30"}, config_type="user"))
         await machine_event.now()
         observations.append(("machine_event_path", bool(machine_event.event_path)))
 
@@ -405,7 +469,6 @@ def test_machine_service_persists_only_derived_config_events(tmp_path, hermetic_
                     config={
                         "CHROME_ISOLATION": "snapshot",
                         "CHROME_USER_DATA_DIR": "/tmp/stale-profile",
-                        "ABX_RUNTIME": "archivebox",
                     },
                     config_type="user",
                 ),
@@ -569,7 +632,7 @@ def test_crawl_runner_empty_plugin_selection_emits_lifecycle_and_seals_crawl(tmp
     assert runner.bus.event_is_child_of(completed_events[0], crawl_events[0])
     assert runner.bus.event_is_child_of(snapshot_events[0], start_events[0])
     assert runner.bus.event_is_child_of(snapshot_completed_events[0], snapshot_events[0])
-    assert any(event.config_type == "user" and event.config.get("ABX_RUNTIME") == "archivebox" for event in machine_events)
+    assert any(event.config_type == "user" for event in machine_events)
 
     crawl.refresh_from_db()
     snapshot = Snapshot.objects.get(crawl=crawl)

archivebox/tests/test_dockerfiles.py

@@ -52,6 +52,11 @@ def _archivebox_install_commands(dockerfile: Path) -> list[str]:
     return re.findall(r"archivebox install ([^\\\n]+)", text)
 
 
+def _bare_archivebox_install_commands(dockerfile: Path) -> list[str]:
+    text = dockerfile.read_text()
+    return re.findall(r"archivebox install(?:\s+2>&1|\s*\\)", text)
+
+
 def _abx_dl_plugin_install_targets(dockerfile: Path) -> set[str]:
     text = dockerfile.read_text()
     commands = re.findall(r"abx-dl plugins --install((?: \\\n|[^\n])*)", text)
@@ -81,3 +86,22 @@ def test_dockerfile_build_installs_disable_release_age_gate() -> None:
         text = (REPO_ROOT / dockerfile_name).read_text()
         assert "ABXPKG_POSTINSTALL_SCRIPTS=True" in text
         assert "ABXPKG_MIN_RELEASE_AGE=0" in text
+
+
+def test_dockerfiles_do_not_queue_optional_binaries_during_validation() -> None:
+    for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
+        assert _bare_archivebox_install_commands(REPO_ROOT / dockerfile_name) == []
+
+
+def test_dockerfiles_clean_build_data_dir_before_init() -> None:
+    for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
+        text = (REPO_ROOT / dockerfile_name).read_text()
+        assert 'find "$DATA_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +' in text
+
+
+def test_dockerfiles_pin_project_binary_paths_for_validation() -> None:
+    for dockerfile_name in ("Dockerfile", "Dockerfile.multistage"):
+        text = (REPO_ROOT / dockerfile_name).read_text()
+        assert 'GIT_BINARY="$LIB_DIR/env/bin/git"' in text
+        assert 'GALLERYDL_BINARY="$LIB_DIR/env/bin/gallery-dl"' in text
+        assert 'FORUMDL_BINARY="$LIB_DIR/env/bin/forum-dl"' in text

archivebox/tests/test_frozen_crawl_config.py

@@ -1,3 +1,6 @@
+import time
+from types import SimpleNamespace
+
 import pytest
 from django.test import RequestFactory
 from django.utils import timezone
@@ -65,6 +68,8 @@ def test_crawl_save_freezes_full_raw_persona_config_and_redacts_public_serializa
     assert crawl.config["USER_AGENT"] == "Frozen UA"
     assert crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
     assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
+    assert "ACTIVE_PERSONA" not in crawl.config
+    assert "DEFAULT_PERSONA" not in crawl.config
     assert "CRAWL_DIR" not in crawl.config
     assert "SNAP_DIR" not in crawl.config
     assert "DEBUG" not in crawl.config
@@ -82,9 +87,10 @@ def test_crawl_save_freezes_full_raw_persona_config_and_redacts_public_serializa
     redacted_runtime_config = get_config(crawl=crawl, redact_sensitive=True)
     assert redacted_runtime_config.USER_AGENT == "Frozen UA"
     assert redacted_runtime_config.TWOCAPTCHA_API_KEY == SENSITIVE_CONFIG_VALUE_REDACTED
-    execution_config = runtime_config.for_crawl_execution()
+    execution_config = runtime_config.for_crawl()
     assert execution_config["DEBUG"] is False
-    assert execution_config["CRAWL_DIR"] == str(crawl.output_dir)
+    assert "CRAWL_DIR" not in execution_config
+    assert "SNAP_DIR" not in execution_config
     assert "SECRET_KEY" not in execution_config
     assert "PUBLIC_ADD_VIEW" not in execution_config
     assert "DATABASE_NAME" not in execution_config
@@ -123,11 +129,143 @@ def test_config_scopes_are_derived_from_section_and_field_metadata():
 
     assert ArchiveBoxConfig.scope_for_key("TIMEOUT") == "crawl_frozen"
     assert ArchiveBoxConfig.scope_for_key("DEBUG") == "crawl_execution"
-    assert ArchiveBoxConfig.scope_for_key("CRAWL_DIR") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("DEFAULT_PERSONA") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("WGET_ENABLED") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("SEARCH_BACKEND_SQLITE_ENABLED") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("SEARCH_BACKEND_ENGINE") == "crawl_execution"
+    assert ArchiveBoxConfig.scope_for_key("WGET_WARC_ENABLED") == "crawl_frozen"
     assert ArchiveBoxConfig.scope_for_key("SECRET_KEY") == "server"
     assert ArchiveBoxConfig.scope_for_key("DATABASE_NAME") == "server"
 
 
+def test_search_backend_engine_derives_default_backend_enabled_without_entering_hook_env():
+    from archivebox.config.common import ArchiveBoxConfig
+
+    default_runtime_config = ArchiveBoxConfig().for_crawl_runtime(extra_context={"snapshot_id": "default-runtime-config"})
+    assert default_runtime_config["SEARCH_BACKEND_RIPGREP_ENABLED"] is True
+    assert default_runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is False
+    assert default_runtime_config["SEARCH_BACKEND_SONIC_ENABLED"] is True
+
+    sqlite_runtime_config = ArchiveBoxConfig(SEARCH_BACKEND_ENGINE="sqlite").for_crawl_runtime(
+        extra_context={"snapshot_id": "sqlite-runtime-config"},
+    )
+    assert sqlite_runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is True
+
+    config = ArchiveBoxConfig(
+        SEARCH_BACKEND_ENGINE="ripgrep",
+        SEARCH_BACKEND_SQLITE_ENABLED=False,
+        SEARCH_BACKEND_SONIC_ENABLED=True,
+        SECRET_KEY="server-secret",
+        DATABASE_NAME="server-db.sqlite3",
+    )
+
+    runtime_config = config.for_crawl_runtime(extra_context={"snapshot_id": "runtime-config"})
+
+    assert "SEARCH_BACKEND_ENGINE" not in runtime_config
+    assert runtime_config["SEARCH_BACKEND_RIPGREP_ENABLED"] is True
+    assert runtime_config["SEARCH_BACKEND_SQLITE_ENABLED"] is False
+    assert runtime_config["SEARCH_BACKEND_SONIC_ENABLED"] is True
+    assert "SECRET_KEY" not in runtime_config
+    assert "DATABASE_NAME" not in runtime_config
+
+
+def test_plugin_selection_enabled_keys_are_derived_from_plugins_not_frozen_or_env_overridden(archivebox_db, monkeypatch):
+    from archivebox.config.common import get_config
+    from archivebox.crawls.models import Crawl
+    from archivebox.plugins.discovery import get_plugin_special_config
+
+    monkeypatch.setenv("ARCHIVEDOTORG_ENABLED", "False")
+    user = _user("frozen-config-enabled-admin")
+    persona = _persona(user, name="Enabled Persona")
+
+    env_default_crawl = Crawl(
+        urls="https://example.com/env-enabled",
+        persona=persona,
+        created_by=user,
+        config={},
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
+    )
+    env_default_crawl.save()
+    env_default_config = get_config(crawl=env_default_crawl, include_machine=False)
+    assert env_default_config.ARCHIVEDOTORG_ENABLED is False
+
+    crawl = Crawl(
+        urls="https://example.com/enabled",
+        persona=persona,
+        created_by=user,
+        config={"PLUGINS": "archivedotorg", "ARCHIVEDOTORG_ENABLED": True, "DEFAULT_PERSONA": "Other"},
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
+    )
+    crawl.save()
+
+    assert crawl.config["PLUGINS"] == "archivedotorg"
+    assert "ARCHIVEDOTORG_ENABLED" not in crawl.config
+    assert "DEFAULT_PERSONA" not in crawl.config
+    runtime_config = get_config(crawl=crawl, include_machine=False)
+    assert runtime_config.ARCHIVEDOTORG_ENABLED is True
+    assert runtime_config.WGET_ENABLED is False
+    assert get_plugin_special_config("archivedotorg", runtime_config)["enabled"] is True
+    assert get_plugin_special_config("wget", runtime_config)["enabled"] is False
+
+    monkeypatch.delenv("ARCHIVEDOTORG_ENABLED")
+
+    Crawl.objects.filter(id=crawl.id).update(
+        config={
+            **crawl.config,
+            "ARCHIVEDOTORG_ENABLED": False,
+            "YTDLP_ENABLED": True,
+            "WGET_ENABLED": True,
+        },
+    )
+    crawl.refresh_from_db()
+    stale_runtime_config = get_config(crawl=crawl, include_machine=False)
+    assert stale_runtime_config.ARCHIVEDOTORG_ENABLED is True
+    assert stale_runtime_config.YTDLP_ENABLED is False
+    assert stale_runtime_config.WGET_ENABLED is False
+
+
+def test_crawl_config_projections_stay_under_hot_path_budget():
+    from archivebox.config.common import ArchiveBoxConfig
+
+    config = ArchiveBoxConfig(TIMEOUT=12, USER_AGENT="Perf UA", CHROME_BINARY="perf-chrome")
+    persona = SimpleNamespace(
+        config={"USER_AGENT": "Persona UA"},
+        get_derived_config=lambda: {
+            "ACTIVE_PERSONA": "Perf Persona",
+            "USER_AGENT": "Persona UA",
+            "CHROME_USER_DATA_DIR": "/tmp/persona/chrome",
+        },
+    )
+    crawl = SimpleNamespace(config={"TIMEOUT": 13, "CHROME_BINARY": "crawl-chrome"})
+    snapshot = SimpleNamespace(config={"TIMEOUT": 14, "CHROME_BINARY": "snapshot-chrome"})
+    runtime_kwargs = {
+        "crawl": crawl,
+        "snapshot": snapshot,
+        "persona": persona,
+        "crawl_output_dir": "/tmp/archivebox/crawls/perf",
+        "snapshot_output_dir": "/tmp/archivebox/crawls/perf/snapshots/example",
+        "extra_context": {"snapshot_id": "perf"},
+    }
+
+    methods = {
+        "for_crawl": config.for_crawl,
+        "for_crawl_frozen": lambda: config.for_crawl_frozen(persona=persona),
+        "for_crawl_runtime": lambda: config.for_crawl_runtime(**runtime_kwargs),
+    }
+    iterations = 250
+    max_average_seconds = 0.020
+
+    for name, method in methods.items():
+        method()
+        started_at = time.perf_counter()
+        for _ in range(iterations):
+            method()
+        average_seconds = (time.perf_counter() - started_at) / iterations
+        assert average_seconds < max_average_seconds, f"{name} averaged {average_seconds * 1000:.3f}ms"
+
+
 def test_api_create_and_cli_add_store_full_frozen_config(archivebox_db):
     from archivebox.api.v1_crawls import CrawlCreateSchema, CrawlSchema, create_crawl
     from archivebox.cli.archivebox_add import add
@@ -201,3 +339,39 @@ def test_schedule_enqueue_refreezes_using_current_template_persona_defaults(arch
     assert "PUBLIC_ADD_VIEW" not in child.config
     assert template.config["USER_AGENT"] == "Initial schedule UA"
     assert template.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
+
+
+def test_crawl_config_backfill_migration_uses_frozen_config_helper(archivebox_db):
+    import importlib
+
+    from django.apps import apps
+    from django.db import connection
+
+    from archivebox.crawls.models import Crawl
+
+    migration = importlib.import_module("archivebox.crawls.migrations.0018_freeze_crawl_config_snapshots")
+    user = _user("frozen-config-migration-admin")
+    persona = _persona(user, name="Migration Persona", user_agent="Migration UA")
+    crawl = Crawl(
+        urls="https://example.com/migration",
+        persona=persona,
+        created_by=user,
+        config={"TIMEOUT": 44, "CHROME_BINARY": "migration-chrome"},
+        status=Crawl.StatusChoices.QUEUED,
+        retry_at=timezone.now(),
+    )
+    crawl.save()
+
+    Crawl.objects.filter(id=crawl.id).update(config={"TIMEOUT": 44, "CHROME_BINARY": "migration-chrome"})
+    migration.freeze_existing_crawl_configs(apps, SimpleNamespace(connection=connection))
+
+    crawl.refresh_from_db()
+    assert crawl.config["TIMEOUT"] == 44
+    assert "CHROME_BINARY" not in crawl.config
+    assert crawl.config["USER_AGENT"] == "Migration UA"
+    assert crawl.config["TWOCAPTCHA_API_KEY"] == SENSITIVE_SECRET
+    assert "ACTIVE_PERSONA" not in crawl.config
+    assert "DEFAULT_PERSONA" not in crawl.config
+    assert "CRAWL_DIR" not in crawl.config
+    assert "SNAP_DIR" not in crawl.config
+    assert "SECRET_KEY" not in crawl.config

archivebox/tests/test_hooks.py

@@ -45,10 +45,11 @@ def create_test_plugin_structure(plugins_dir: Path) -> None:
 
 def run_plugin_discovery_subprocess(tmp_path: Path, plugins_dir: Path, script: str):
     env = os.environ.copy()
-    env["ARCHIVEBOX_USER_PLUGINS_DIR"] = str(plugins_dir)
     data_dir = tmp_path / "data"
     data_dir.mkdir()
-    env["DATA_DIR"] = str(data_dir)
+    cwd_plugins_dir = data_dir / "custom_plugins"
+    if plugins_dir != cwd_plugins_dir:
+        shutil.copytree(plugins_dir, cwd_plugins_dir)
     env["PYTHONPATH"] = str(REPO_ROOT) + (os.pathsep + env["PYTHONPATH"] if env.get("PYTHONPATH") else "")
     subprocess_script = "\n".join(
         [
@@ -205,7 +206,7 @@ class TestRequiredBinaryConfigHandling:
         assert binary_name == "wget2"
 
     def test_binary_env_var_empty_default(self):
-        """Empty configured values should fall back to config defaults."""
+        """Empty configured binary values should keep the schema default."""
         configured_binary = ""
         if configured_binary:
             binary_name = configured_binary
@@ -736,7 +737,6 @@ print(json.dumps({
         hook_path,
         output_dir,
         config={
-            "DATA_DIR": str(tmp_path),
             "LIB_DIR": str(lib_dir),
             "NODE_PATH": configured_node_path,
         },

archivebox/tests/test_machine_models.py

@@ -90,7 +90,6 @@ class TestMachineModel:
 
     def test_machine_config_save_heals_json_encoded_string_values(self, machine):
         machine.config = {
-            "PERSONAS_DIR": '"/data/personas"',
             "EXTRA_CONTEXT": 'prefix "inner" suffix',
             "USER_AGENT": '"ArchiveBox \\"Quoted\\" Agent"',
         }
@@ -98,7 +97,6 @@ class TestMachineModel:
 
         machine.refresh_from_db()
 
-        assert machine.config["PERSONAS_DIR"] == "/data/personas"
         assert machine.config["EXTRA_CONTEXT"] == 'prefix "inner" suffix'
         assert machine.config["USER_AGENT"] == 'ArchiveBox "Quoted" Agent'
 

archivebox/tests/test_misc_checks.py

@@ -1,6 +1,10 @@
 import os
 import signal
 
+import pytest
+
+from archivebox.core.shutdown_util import foreground_shutdown_signals
+from archivebox.core.shutdown_util import raise_if_shutdown_requested
 from archivebox.misc.checks import _migration_interrupt_message
 from archivebox.misc.checks import _exit_on_migration_interrupt
 
@@ -38,3 +42,16 @@ def test_migration_interrupt_handler_exits_for_sigint_and_sigterm(monkeypatch):
         else:
             raise AssertionError(f"{sig.name} should exit during migration auto-apply")
         assert signal.getsignal(sig) == previous_handler
+
+
+def test_nested_foreground_signal_state_propagates_to_outer_context():
+    with foreground_shutdown_signals(first_signal_message=None) as outer_state:
+        try:
+            with foreground_shutdown_signals(first_signal_message=None):
+                os.kill(os.getpid(), signal.SIGTERM)
+        except KeyboardInterrupt:
+            pass
+
+        assert outer_state.signal_name == "SIGTERM"
+        with pytest.raises(KeyboardInterrupt):
+            raise_if_shutdown_requested()

archivebox/tests/test_orm_helpers.py

@@ -17,23 +17,20 @@ def use_archivebox_db(path: str | Path = ".") -> Iterator[None]:
     original_name = connection.settings_dict["NAME"]
     original_database_name = connections.databases["default"]["NAME"]
     original_setting_name = settings.DATABASES["default"]["NAME"]
-    original_connection = getattr(connections._connections, "default", None)
+    original_connection = connections._connections.default
     db_path = str(archivebox_db_path(path))
 
     connection.close()
     connection.settings_dict["NAME"] = db_path
     connections.databases["default"]["NAME"] = db_path
     settings.DATABASES["default"]["NAME"] = db_path
-    if original_connection is not None:
-        delattr(connections._connections, "default")
+    del connections._connections.default
     try:
         yield
     finally:
         connections["default"].close()
         connections.databases["default"]["NAME"] = original_database_name
         settings.DATABASES["default"]["NAME"] = original_setting_name
-        if hasattr(connections._connections, "default"):
-            delattr(connections._connections, "default")
-        if original_connection is not None:
-            original_connection.settings_dict["NAME"] = original_name
-            setattr(connections._connections, "default", original_connection)
+        del connections._connections.default
+        original_connection.settings_dict["NAME"] = original_name
+        connections._connections.default = original_connection

archivebox/tests/test_persona_admin.py

@@ -5,6 +5,7 @@ from django.contrib.auth import get_user_model
 from django.contrib.auth.models import UserManager
 from django.urls import reverse
 
+from archivebox.config.common import ArchiveBoxConfig
 from archivebox.personas.importers import (
     discover_persona_template_profiles,
     import_persona_from_source,
@@ -175,20 +176,20 @@ def test_persona_admin_add_post_runs_shared_importer(client, admin_user):
 
 
 def test_persona_admin_saves_typed_plugin_config(client, admin_user):
-    from archivebox.config.common import get_config
-
     client.login(username="personaadmin", password="testpassword")
     add_response = client.get(reverse("admin:personas_persona_add"), HTTP_HOST=ADMIN_HOST)
     add_form = add_response.context["adminform"].form
-    personas_dir_fields = [
-        field
-        for group in add_form.plugin_groups
-        for card in group["plugins"]
-        for field in card["config_fields"]
-        if field["key"] == "PERSONAS_DIR"
-    ]
-    assert personas_dir_fields
-    assert {field["value"] for field in personas_dir_fields} == {str(get_config().PERSONAS_DIR)}
+    exposed_config_keys = {field["key"] for group in add_form.plugin_groups for card in group["plugins"] for field in card["config_fields"]}
+    assert not {key for key in exposed_config_keys if ArchiveBoxConfig.scope_for_key(key) == "crawl_execution"}
+    assert (
+        not {
+            "ARCHIVE_DIR",
+            "USERS_DIR",
+            "PERSONAS_DIR",
+            "CUSTOM_TEMPLATES_DIR",
+        }
+        & exposed_config_keys
+    )
 
     response = client.post(
         reverse("admin:personas_persona_add"),
@@ -216,7 +217,6 @@ def test_persona_config_save_heals_json_encoded_string_values(admin_user):
         name="QuotedConfigPersona",
         created_by=admin_user,
         config={
-            "PERSONAS_DIR": '"/data/personas"',
             "EXTRA_CONTEXT": 'prefix "inner" suffix',
             "USER_AGENT": '"ArchiveBox \\"Quoted\\" Agent"',
         },
@@ -224,6 +224,5 @@ def test_persona_config_save_heals_json_encoded_string_values(admin_user):
 
     persona.refresh_from_db()
 
-    assert persona.config["PERSONAS_DIR"] == "/data/personas"
     assert persona.config["EXTRA_CONTEXT"] == 'prefix "inner" suffix'
     assert persona.config["USER_AGENT"] == 'ArchiveBox "Quoted" Agent'

archivebox/tests/test_persona_runtime.py

@@ -218,7 +218,7 @@ def test_get_config_treats_missing_persona_id_as_null(initialized_archive):
     assert payload["persona_id"] == "None"
 
 
-def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(initialized_archive):
+def test_get_config_resolves_parent_scopes_for_snapshot_runtime(initialized_archive):
     script = textwrap.dedent(
         """
         import json
@@ -237,6 +237,7 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
         from archivebox.crawls.models import Crawl
         from archivebox.machine.models import Machine
         from archivebox.personas.models import Persona
+        from archivebox.services.runner import CrawlRunner
 
         CONSTANTS.CONFIG_FILE.write_text('[ARCHIVING_CONFIG]\\nTIMEOUT=11\\nCHROME_BINARY=file-chrome\\n')
 
@@ -254,6 +255,8 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
             persona_id=persona.id,
             config={'TIMEOUT': 44, 'CHROME_BINARY': 'crawl-chrome'},
         )
+        persona.config = {'TIMEOUT': 99, 'CHROME_BINARY': 'persona-chrome-updated'}
+        persona.save(update_fields=['config'])
         snapshot = Snapshot.objects.create(
             url='https://example.com',
             crawl=crawl,
@@ -262,7 +265,6 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
         result = ArchiveResult.objects.create(
             snapshot=snapshot,
             plugin='title',
-            config={'TIMEOUT': 66, 'CHROME_BINARY': 'archiveresult-chrome'},
         )
 
         env_config = get_config(include_machine=False)
@@ -270,8 +272,10 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
         persona_config = get_config(persona=persona)
         crawl_config = get_config(crawl=crawl)
         snapshot_config = get_config(snapshot=snapshot)
-        result_config = get_config(archiveresult=result)
-        override_config = get_config(archiveresult=result, overrides={'TIMEOUT': 77, 'CHROME_BINARY': 'override-chrome'})
+        override_config = get_config(snapshot=snapshot, overrides={'TIMEOUT': 77, 'CHROME_BINARY': 'override-chrome'})
+        runner = CrawlRunner(crawl, selected_plugins=['title'], show_progress=False)
+        runner.load_run_state()
+        runtime_config = runner.load_snapshot_payload(str(snapshot.id))['config']
 
         print(json.dumps({
             'env': [env_config.TIMEOUT, env_config.CHROME_BINARY],
@@ -279,13 +283,12 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
             'persona': [persona_config.TIMEOUT, persona_config.CHROME_BINARY],
             'crawl': [crawl_config.TIMEOUT, crawl_config.CHROME_BINARY],
             'snapshot': [snapshot_config.TIMEOUT, snapshot_config.CHROME_BINARY],
-            'archiveresult': [result_config.TIMEOUT, result_config.CHROME_BINARY],
             'override': [override_config.TIMEOUT, override_config.CHROME_BINARY],
-            'snap_dir': str(result_config.SNAP_DIR),
+            'snap_dir': str(runtime_config['SNAP_DIR']),
             'expected_snap_dir': str(snapshot.output_dir),
-            'crawl_dir': str(result_config.CRAWL_DIR),
+            'crawl_dir': str(runtime_config['CRAWL_DIR']),
             'expected_crawl_dir': str(crawl.output_dir),
-            'active_persona': result_config.ACTIVE_PERSONA,
+            'active_persona': runtime_config['ACTIVE_PERSONA'],
         }, default=str))
         """,
     )
@@ -296,11 +299,10 @@ def test_get_config_resolves_parent_scopes_when_only_archiveresult_is_passed(ini
     payload = json.loads(stdout.strip().splitlines()[-1])
     assert payload["env"] == [22, "env-chrome"]
     assert payload["machine"] == [22, "machine-chrome"]
-    assert payload["persona"] == [33, "persona-chrome"]
-    assert payload["crawl"] == [44, "crawl-chrome"]
-    assert payload["snapshot"] == [55, "snapshot-chrome"]
-    assert payload["archiveresult"] == [66, "archiveresult-chrome"]
-    assert payload["override"] == [77, "override-chrome"]
+    assert payload["persona"] == [99, "persona-chrome-updated"]
+    assert payload["crawl"] == [44, "persona-chrome-updated"]
+    assert payload["snapshot"] == [55, "persona-chrome-updated"]
+    assert payload["override"] == [77, "persona-chrome-updated"]
     assert payload["snap_dir"] == payload["expected_snap_dir"]
     assert payload["crawl_dir"] == payload["expected_crawl_dir"]
     assert payload["active_persona"] == "StackPersona"

archivebox/tests/test_search.py

@@ -4,7 +4,6 @@ import socket
 import subprocess
 import sys
 import time
-from pathlib import Path
 from types import SimpleNamespace
 from urllib.parse import urlencode
 
@@ -97,8 +96,6 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
     os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] = "old-host"
     config = AttrDict(
         {
-            "DATA_DIR": tmp_path,
-            "USERS_DIR": tmp_path / "archive" / "users",
             "SEARCH_BACKEND_ENGINE": "sonic",
             "SEARCH_BACKEND_SONIC_HOST_NAME": "sonic",
             "SEARCH_BACKEND_SONIC_PORT": 1491,
@@ -109,8 +106,6 @@ def test_search_backend_env_exposes_resolved_runtime_config(tmp_path):
 
     try:
         with search_backend_env(config=config):
-            assert os.environ["DATA_DIR"] == str(tmp_path)
-            assert os.environ["SNAP_DIR"] == str(Path(tmp_path) / "archive" / "users")
             assert os.environ["SEARCH_BACKEND_ENGINE"] == "sonic"
             assert os.environ["SEARCH_BACKEND_SONIC_HOST_NAME"] == "sonic"
             assert os.environ["SEARCH_BACKEND_SONIC_PORT"] == "1491"
@@ -465,7 +460,6 @@ class TestSearchBackendsE2E:
             merged_env = os.environ.copy()
             merged_env.update(
                 {
-                    "DATA_DIR": str(data_dir),
                     "USE_COLOR": "False",
                     "SHOW_PROGRESS": "False",
                     "SAVE_WARC": "False",
@@ -505,7 +499,6 @@ class TestSearchBackendsE2E:
         sonic_env = os.environ.copy()
         sonic_env.update(
             {
-                "DATA_DIR": str(data_dir),
                 "USE_COLOR": "False",
                 "SHOW_PROGRESS": "False",
                 "SEARCH_BACKEND_ENGINE": "sonic",

archivebox/tests/test_shutdown_util.py

@@ -0,0 +1,76 @@
+import signal
+
+import pytest
+
+from archivebox.cli import archivebox_run
+from archivebox.core import shutdown_util
+
+
+def test_foreground_shutdown_second_signal_exits_immediately(monkeypatch):
+    def fake_exit(code):
+        raise SystemExit(code)
+
+    monkeypatch.setattr(shutdown_util.os, "_exit", fake_exit)
+
+    with shutdown_util.foreground_shutdown_signals() as state:
+        handler = signal.getsignal(signal.SIGTERM)
+
+        with pytest.raises(KeyboardInterrupt):
+            handler(signal.SIGTERM, None)
+        assert state.signal_name == "SIGTERM"
+
+        with pytest.raises(SystemExit) as err:
+            handler(signal.SIGTERM, None)
+        assert err.value.code == 130
+
+
+def test_foreground_shutdown_can_request_cooperative_shutdown_without_raising(monkeypatch):
+    def fake_exit(code):
+        raise SystemExit(code)
+
+    seen = []
+    monkeypatch.setattr(shutdown_util.os, "_exit", fake_exit)
+
+    with shutdown_util.foreground_shutdown_signals(
+        first_signal_message=None,
+        on_signal=seen.append,
+        raise_on_first_signal=False,
+    ) as state:
+        handler = signal.getsignal(signal.SIGTERM)
+
+        handler(signal.SIGTERM, None)
+        assert state.signal_name == "SIGTERM"
+        assert seen == [signal.SIGTERM]
+
+        with pytest.raises(SystemExit) as err:
+            handler(signal.SIGTERM, None)
+        assert err.value.code == 130
+
+
+def test_daemon_runner_signal_exit_is_unexpected_for_supervisor(monkeypatch):
+    def fake_exit(code):
+        raise SystemExit(code)
+
+    monkeypatch.setattr(archivebox_run.os, "_exit", fake_exit)
+
+    with pytest.raises(SystemExit) as err:
+        archivebox_run._exit_daemon_runner_on_signal(signal.SIGTERM)
+
+    assert err.value.code == 143
+
+
+def test_crawl_runner_daemon_signal_exits_before_async_cleanup(monkeypatch):
+    from archivebox.services import runner as runner_module
+    from archivebox.services.runner import CrawlRunner
+
+    def fake_exit(code):
+        raise SystemExit(code)
+
+    runner = object.__new__(CrawlRunner)
+    monkeypatch.setenv("ARCHIVEBOX_RUNNER_DAEMON", "1")
+    monkeypatch.setattr(runner_module.os, "_exit", fake_exit)
+
+    with pytest.raises(SystemExit) as err:
+        runner._request_abort_from_signal(signal.SIGTERM)
+
+    assert err.value.code == 143

archivebox/tests/test_test_harness.py

@@ -18,15 +18,17 @@ def test_in_process_archivebox_config_uses_temp_data_dir():
     from archivebox.config import CONSTANTS
     from archivebox.config.common import get_config
 
-    data_dir = Path(os.environ["DATA_DIR"]).resolve()
+    data_dir = CONSTANTS.DATA_DIR.resolve()
     assert data_dir == Path.cwd().resolve()
     assert test_harness.REPO_ROOT not in data_dir.parents
     assert CONSTANTS.DATA_DIR != test_harness.REPO_ROOT
 
     config = get_config(include_machine=False)
-    assert config.DATA_DIR == data_dir
-    assert config.ARCHIVE_DIR == data_dir / "archive"
-    assert config.USERS_DIR == data_dir / "archive" / "users"
+    assert "DATA_DIR" not in config
+    assert "ARCHIVE_DIR" not in config
+    assert "USERS_DIR" not in config
+    assert CONSTANTS.ARCHIVE_DIR == data_dir / "archive"
+    assert CONSTANTS.USERS_DIR == data_dir / "archive" / "users"
 
 
 def test_cli_helpers_reject_repo_root_runtime_paths():

archivebox/tests/test_ui_add_view.py

@@ -3,10 +3,11 @@ import pytest
 from django.contrib.auth import get_user_model
 from django.urls import reverse
 
-from archivebox.config.common import get_config
+from archivebox.config.common import ArchiveBoxConfig
 from archivebox.core.models import Snapshot, Tag
 from archivebox.crawls.models import Crawl
 from archivebox.personas.models import Persona
+from archivebox.services.runner import CrawlRunner
 from archivebox.workers.models import RETRY_AT_MAX
 
 
@@ -74,15 +75,28 @@ def test_add_view_admin_renders_plugin_config_grid(client, admin_user, monkeypat
     assert b">Docs</a>" in response.content
     assert b"https://github.com/ArchiveBox/abx-plugins/tree/main/abx_plugins/plugins/" in response.content
     assert b"https://archivebox.github.io/abx-plugins/#" in response.content
-    personas_dir_fields = [
-        field
+    assert not any(
+        field["key"].endswith("_BINARY") for group in form.plugin_groups for card in group["plugins"] for field in card["config_fields"]
+    )
+    assert not any(
+        field["key"] == f"{card['name'].upper()}_ENABLED"
         for group in form.plugin_groups
         for card in group["plugins"]
         for field in card["config_fields"]
-        if field["key"] == "PERSONAS_DIR"
-    ]
-    assert personas_dir_fields
-    assert {field["value"] for field in personas_dir_fields} == {str(get_config().PERSONAS_DIR)}
+    )
+    exposed_config_keys = {field["key"] for group in form.plugin_groups for card in group["plugins"] for field in card["config_fields"]}
+    assert not {key for key in exposed_config_keys if ArchiveBoxConfig.scope_for_key(key) == "crawl_execution"}
+    assert (
+        not {
+            "ARCHIVE_DIR",
+            "USERS_DIR",
+            "PERSONAS_DIR",
+            "CUSTOM_TEMPLATES_DIR",
+        }
+        & exposed_config_keys
+    )
+    assert b"plugin_config__chrome__CHROME_BINARY" not in response.content
+    assert b"plugin_config__wget__WGET_ENABLED" not in response.content
 
 
 def test_add_view_embeds_selected_persona_config_for_ui_hydration(client, admin_user, monkeypatch):
@@ -127,6 +141,20 @@ def test_add_view_public_only_lists_public_personas(client, admin_user, monkeypa
     assert {"NODE_BINARY", "TWOCAPTCHA_API_KEY"}.isdisjoint(persona_config_map["Default"]["effective_config"])
 
 
+def test_persona_config_grid_allows_binary_fields(client, admin_user):
+    client.force_login(admin_user)
+    persona = Persona.objects.create(name="Binary Persona", created_by=admin_user)
+
+    response = client.get(
+        reverse("admin:personas_persona_change", args=[persona.pk]),
+        HTTP_HOST=ADMIN_HOST,
+    )
+
+    assert response.status_code == 200
+    assert b"plugin_config__chrome__CHROME_BINARY" in response.content
+    assert b"plugin_config__wget__WGET_ENABLED" in response.content
+
+
 def test_add_view_hides_search_backend_plugins(client, monkeypatch):
     monkeypatch.setenv("PUBLIC_ADD_VIEW", "true")
     monkeypatch.setenv("SEARCH_BACKEND_ENGINE", "sqlite")
@@ -253,11 +281,14 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
     crawl = Crawl.objects.order_by("-created_at").first()
     assert crawl is not None
     assert crawl.persona_id == private_persona.id
-    assert crawl.config["ACTIVE_PERSONA"] == "Private"
+    assert "ACTIVE_PERSONA" not in crawl.config
     assert crawl.resolve_persona() == private_persona
-    runtime_config = get_config(crawl=crawl)
-    assert runtime_config.ACTIVE_PERSONA == "Private"
-    assert runtime_config.COOKIES_FILE == private_cookies_file
+    snapshot = Snapshot.objects.create(url="https://example.com/private", crawl=crawl)
+    runner = CrawlRunner(crawl, selected_plugins=["title"], show_progress=False)
+    runner.load_run_state()
+    runtime_config = runner.load_snapshot_payload(str(snapshot.id))["config"]
+    assert runtime_config["ACTIVE_PERSONA"] == "Private"
+    assert runtime_config["COOKIES_FILE"] == str(private_cookies_file)
 
 
 def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatch):
@@ -282,9 +313,11 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc
             "permissions": "public",
             "start_paused": "",
             "main_plugins": ["wget"],
+            "plugin_config__wget__WGET_ENABLED": "false",
             "plugin_config__wget__WGET_TIMEOUT": "77",
             "plugin_config__wget__WGET_WARC_ENABLED": "false",
-            "config": "{}",
+            "plugin_config__chrome__CHROME_BINARY": "/tmp/malicious-chrome",
+            "config": '{"NODE_BINARY": "/tmp/malicious-node", "WGET_ENABLED": false}',
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -296,6 +329,9 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user, monkeypatc
     assert crawl.config["PLUGINS"] == "wget"
     assert crawl.config["WGET_TIMEOUT"] == 77
     assert crawl.config["WGET_WARC_ENABLED"] is False
+    assert "WGET_ENABLED" not in crawl.config
+    assert "CHROME_BINARY" not in crawl.config
+    assert "NODE_BINARY" not in crawl.config
 
 
 def test_add_view_public_submission_ignores_plugin_and_custom_config(client, admin_user, monkeypatch):

archivebox/tests/test_ui_add_view_runtime.py

@@ -14,6 +14,7 @@ from .conftest import (
     get_depth_counts,
     get_free_port,
     init_archive,
+    run_archivebox_cmd_cwd,
     start_server,
     stop_server,
     wait_for_http,
@@ -22,6 +23,133 @@ from .conftest import (
 pytestmark = pytest.mark.django_db(transaction=True)
 
 
+@pytest.mark.timeout(180)
+def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_site):
+    os.chdir(tmp_path)
+    init_archive(tmp_path)
+
+    port = get_free_port()
+    env = build_test_env(
+        port,
+        PLUGINS="wget",
+        PUBLIC_ADD_VIEW="True",
+        PYTEST_CURRENT_TEST="",
+    )
+    create_admin_and_token(tmp_path)
+
+    try:
+        start_server(tmp_path, env=env, port=port)
+        _wait_for_worker_state(tmp_path, "worker_runner", "RUNNING")
+        _stop_worker(tmp_path, "worker_runner")
+        assert _worker_state(tmp_path, "worker_runner") != "RUNNING"
+
+        session, csrf_token = _login_to_add_view(port)
+        response = session.post(
+            f"http://127.0.0.1:{port}/add/",
+            headers={"Host": f"admin.archivebox.localhost:{port}", "Referer": f"http://admin.archivebox.localhost:{port}/add/"},
+            data={
+                "url": recursive_test_site["root_url"],
+                "depth": "0",
+                "max_urls": "1",
+                "crawl_max_size": "0",
+                "snapshot_max_size": "0",
+                "main_plugins": ["wget"],
+                "tag": "restart-supervised-runner",
+                "url_filters_allowlist": r"127\.0\.0\.1[:/].*",
+                "url_filters_denylist": "",
+                "schedule": "",
+                "notes": "restart stopped supervised runner",
+                "persona": "Default",
+                "permissions": "public",
+                "start_paused": "",
+                "config": "{}",
+                "csrfmiddlewaretoken": csrf_token,
+            },
+            timeout=10,
+            allow_redirects=False,
+        )
+        assert response.status_code in (302, 303), response.text
+
+        _wait_for_worker_state(tmp_path, "worker_runner", "RUNNING")
+        with use_archivebox_db(tmp_path):
+            crawl = Crawl.objects.order_by("-created_at").first()
+            assert crawl is not None
+            assert crawl.tags_str == "restart-supervised-runner"
+            assert crawl.urls == recursive_test_site["root_url"]
+    finally:
+        stop_server(tmp_path)
+
+
+def _login_to_add_view(port: int) -> tuple[requests.Session, str]:
+    session = requests.Session()
+    wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
+    login_page = session.get(
+        f"http://127.0.0.1:{port}/admin/login/",
+        headers={"Host": f"admin.archivebox.localhost:{port}"},
+        timeout=10,
+    )
+    assert login_page.status_code == 200
+    csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
+    assert csrf_match, login_page.text[:500]
+    login_response = session.post(
+        f"http://127.0.0.1:{port}/admin/login/",
+        headers={"Host": f"admin.archivebox.localhost:{port}", "Referer": f"http://admin.archivebox.localhost:{port}/admin/login/"},
+        data={
+            "username": "apitestadmin",
+            "password": "testpass123",
+            "csrfmiddlewaretoken": csrf_match.group(1),
+            "next": "/add/",
+        },
+        timeout=10,
+        allow_redirects=False,
+    )
+    assert login_response.status_code in (302, 303), login_response.text
+    add_page = wait_for_http(port, host=f"admin.archivebox.localhost:{port}", path="/add/")
+    assert add_page.status_code == 200
+    add_csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', add_page.text)
+    assert add_csrf_match, add_page.text[:500]
+    return session, add_csrf_match.group(1)
+
+
+def _worker_state(cwd, worker_name: str) -> str | None:
+    script = f"""
+import json
+from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
+supervisor = get_existing_supervisord_process()
+worker = get_worker(supervisor, {worker_name!r}) if supervisor else None
+print(json.dumps(worker))
+"""
+    stdout, stderr, returncode = run_archivebox_cmd_cwd(["manage", "shell", "-c", script], cwd=cwd, timeout=60)
+    assert returncode == 0, stderr or stdout
+    import json
+
+    worker = json.loads(stdout.strip().splitlines()[-1])
+    return worker.get("statename") if worker else None
+
+
+def _stop_worker(cwd, worker_name: str) -> None:
+    script = f"""
+from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_worker
+supervisor = get_existing_supervisord_process()
+assert supervisor is not None
+stop_worker(supervisor, {worker_name!r})
+print("stopped")
+"""
+    stdout, stderr, returncode = run_archivebox_cmd_cwd(["manage", "shell", "-c", script], cwd=cwd, timeout=60)
+    assert returncode == 0, stderr or stdout
+
+
+def _wait_for_worker_state(cwd, worker_name: str, statename: str, timeout: int = 45) -> None:
+    deadline = time.time() + timeout
+    state = None
+    while time.time() < deadline:
+        state = _worker_state(cwd, worker_name)
+        if state == statename:
+            return
+        time.sleep(1)
+    raise AssertionError(f"Timed out waiting for {worker_name}={statename}, last state={state}")
+
+
 @pytest.mark.timeout(180)
 def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_site):
     os.chdir(tmp_path)

archivebox/tests/test_urls.py

@@ -73,7 +73,7 @@ def _build_script(body: str) -> str:
     from archivebox.core.middleware import ADMIN_LOGIN_HINT_COOKIE
 
     def response_body(resp):
-        if getattr(resp, "streaming", False):
+        if resp.streaming:
             return b"".join(resp.streaming_content)
         return resp.content
 
@@ -732,6 +732,67 @@ class TestUrlRouting:
             },
         )
 
+    def test_subdomain_replay_assets_route_without_base_url(self) -> None:
+        chrome_extensions_dir = self.data_dir / "test-chrome-extensions"
+        self._run(
+            """
+            snapshot = get_snapshot()
+            snapshot_host = get_snapshot_host(str(snapshot.id))
+            extension_dir = Path(SERVER_CONFIG.CHROME_EXTENSIONS_DIR) / "test__archivewebpage"
+            extension_dir.mkdir(parents=True, exist_ok=True)
+            (extension_dir / "ui.js").write_text("window.__archivebox_replay_ui__ = true;\\n", encoding="utf-8")
+            (extension_dir / "sw.js").write_text("self.__archivebox_replay_sw__ = true;\\n", encoding="utf-8")
+
+            client = Client()
+            resp = client.get("/replay/ui.js", HTTP_HOST=snapshot_host)
+            body = response_body(resp).decode("utf-8", "ignore")
+
+            assert resp.status_code == 200
+            assert resp["Content-Type"].startswith("application/javascript")
+            assert "window.__archivebox_replay_ui__" in body
+
+            print("OK")
+            """,
+            mode="safe-subdomains-fullreplay",
+            env_overrides={
+                "BIND_ADDR": "127.0.0.1:8766",
+                "BASE_URL": "",
+                "CHROME_EXTENSIONS_DIR": str(chrome_extensions_dir),
+            },
+        )
+
+    def test_subdomain_replay_assets_use_derived_chromewebstore_extensions_dir(self) -> None:
+        lib_dir = self.data_dir / "test-lib"
+        self._run(
+            """
+            snapshot = get_snapshot()
+            snapshot_host = get_snapshot_host(str(snapshot.id))
+            expected_extensions_dir = Path(SERVER_CONFIG.LIB_DIR) / "chromewebstore" / "extensions"
+            assert Path(SERVER_CONFIG.CHROME_EXTENSIONS_DIR).resolve() == expected_extensions_dir.resolve()
+
+            extension_dir = Path(SERVER_CONFIG.CHROME_EXTENSIONS_DIR) / "test__archivewebpage"
+            extension_dir.mkdir(parents=True, exist_ok=True)
+            (extension_dir / "ui.js").write_text("window.__archivebox_replay_ui_from_lib__ = true;\\n", encoding="utf-8")
+            (extension_dir / "sw.js").write_text("self.__archivebox_replay_sw_from_lib__ = true;\\n", encoding="utf-8")
+
+            client = Client()
+            resp = client.get("/replay/ui.js", HTTP_HOST=snapshot_host)
+            body = response_body(resp).decode("utf-8", "ignore")
+
+            assert resp.status_code == 200
+            assert resp["Content-Type"].startswith("application/javascript")
+            assert "window.__archivebox_replay_ui_from_lib__" in body
+
+            print("OK")
+            """,
+            mode="safe-subdomains-fullreplay",
+            env_overrides={
+                "BIND_ADDR": "127.0.0.1:8766",
+                "BASE_URL": "",
+                "LIB_DIR": str(lib_dir),
+            },
+        )
+
     def test_onedomain_base_url_overrides_are_preserved_for_external_links(self) -> None:
         self._run(
             """

archivebox/workers/models.py

@@ -3,11 +3,13 @@ __package__ = "archivebox.workers"
 import inspect
 import logging
 
-from typing import Any, ClassVar
+from typing import Any, ClassVar, Protocol, cast
 from collections.abc import Iterable
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
-from statemachine.mixins import MachineMixin
+from statemachine.callbacks import SPECS_ALL
+from statemachine.dispatcher import Listener, Listeners
+from statemachine.graph import iterate_states_and_transitions
 
 from django.db import models
 from django.core import checks
@@ -45,7 +47,15 @@ ObjectState = State | str
 ObjectStateList = Iterable[ObjectState]
 
 
-class BaseModelWithStateMachine(models.Model, MachineMixin):
+class ModelStateMachine(Protocol):
+    def tick(self) -> Any: ...
+
+    def pause_requested(self) -> Any: ...
+
+    def resume_requested(self) -> Any: ...
+
+
+class BaseModelWithStateMachine(models.Model):
     StatusChoices: ClassVar[type[DefaultStatusChoices]]
 
     # status: models.CharField
@@ -54,7 +64,8 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
     state_machine_name: str | None = None
     state_field_name: str
     state_machine_attr: str = "sm"
-    bind_events_as_methods: bool = True
+    bind_events_as_methods: bool = False
+    warn_on_save_outside_runner: ClassVar[bool] = True
 
     active_state: ObjectState
     retry_at_field_name: str
@@ -63,6 +74,31 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         app_label = "workers"
         abstract = True
 
+    @property
+    def sm(self) -> StateMachine:
+        """Build the python-statemachine wrapper only at transition callsites.
+
+        This model is loaded by high-volume paths that do not drive lifecycle
+        transitions: admin lists, progress polling, index-only maintenance, and
+        bulk recovery scans all instantiate thousands of rows just to read or
+        update ordinary columns.  python-statemachine setup is correct but not
+        free: it creates per-instance state wrappers, callback registries,
+        queues, locks, and callback adapters.  Paying that cost from Django's
+        model __init__ made plain ORM materialization scale with state-machine
+        setup instead of row decoding.
+
+        ArchiveBox drives lifecycle transitions explicitly through `.sm`
+        (`snapshot.sm.tick()`, `crawl.sm.seal()`, etc.), so the machine can be
+        cached on first use without changing the state model.  Code that only
+        needs database fields never constructs one.
+        """
+        try:
+            machine = vars(self)["_archivebox_state_machine"]
+        except KeyError:
+            machine = self.StateMachineClass(self, state_field=self.state_field_name)
+            vars(self)["_archivebox_state_machine"] = machine
+        return cast(StateMachine, machine)
+
     @classmethod
     def status_counts(cls, queryset: models.QuerySet | None = None, statuses: Iterable[str] | None = None) -> dict[str, int]:
         """Count requested statuses with separate indexed COUNT probes.
@@ -262,7 +298,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         from archivebox.machine.models import Process
 
         process = Process.current()
-        if process.process_type != Process.TypeChoices.ORCHESTRATOR:
+        if self.warn_on_save_outside_runner and process.process_type != Process.TypeChoices.ORCHESTRATOR:
             root_type = getattr(process.root, "process_type", None)
             if root_type != Process.TypeChoices.ORCHESTRATOR:
                 caller = "<unknown>"
@@ -302,42 +338,30 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         super().save(*args, **kwargs)
 
     def pause(self, *, save: bool = True) -> bool:
-        paused_state = getattr(self.StatusChoices, "PAUSED", None)
-        if paused_state is None:
+        try:
+            paused_state = self.StatusChoices.PAUSED
+        except AttributeError:
             return False
         if self.STATE in self.FINAL_STATES or self.is_paused:
             return False
         if save:
-            transition = getattr(getattr(self, self.state_machine_attr, None), "pause_requested", None)
-            if callable(transition):
-                transition()
-                self.refresh_from_db()
-                return self.is_paused
-            previous_state = self.STATE
-            self.STATE = paused_state
-            self.RETRY_AT = RETRY_AT_MAX
-            updated = self.safe_update(
-                {
-                    self.state_field_name: paused_state,
-                    self.retry_at_field_name: RETRY_AT_MAX,
-                },
-                extra_filter={self.state_field_name: previous_state},
-            )
-            return updated
+            cast(ModelStateMachine, self.sm).pause_requested()
+            self.refresh_from_db()
+            return self.is_paused
         self.STATE = paused_state
         self.RETRY_AT = RETRY_AT_MAX
         return True
 
     def resume(self, *, when: datetime | None = None, save: bool = True) -> bool:
-        paused_state = getattr(self.StatusChoices, "PAUSED", None)
-        if paused_state is None:
+        try:
+            paused_state = self.StatusChoices.PAUSED
+        except AttributeError:
             return False
         if not self.is_paused:
             return False
         if save:
-            transition = getattr(getattr(self, self.state_machine_attr, None), "resume_requested", None)
-            if callable(transition) and when is None:
-                transition()
+            if when is None:
+                cast(ModelStateMachine, self.sm).resume_requested()
                 self.refresh_from_db()
                 return self.STATE == self.StatusChoices.QUEUED
             self.STATE = self.StatusChoices.QUEUED
@@ -416,7 +440,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         )
         if updated == 1:
             obj.RETRY_AT = lock_until
-            obj.modified_at = now
+            cast(Any, obj).modified_at = now
         return updated == 1
 
     def claim_processing_lock(self, lock_seconds: int = 60) -> bool:
@@ -452,10 +476,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         if not self.claim_processing_lock(lock_seconds=lock_seconds):
             return False
 
-        tick = getattr(getattr(self, self.state_machine_attr, None), "tick", None)
-        if not callable(tick):
-            raise TypeError(f"{type(self).__name__}.{self.state_machine_attr}.tick() must be callable")
-        tick()
+        cast(ModelStateMachine, self.sm).tick()
         self.refresh_from_db()
         return True
 
@@ -545,9 +566,9 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
 
     @classproperty
     def StateMachineClass(cls) -> type[StateMachine]:
-        """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
+        """Get the StateMachine class for the given django Model."""
 
-        model_state_machine_name = getattr(cls, "state_machine_name", None)
+        model_state_machine_name = cls.state_machine_name
         if model_state_machine_name:
             StateMachineCls = registry.get_machine_cls(model_state_machine_name)
             assert issubclass(StateMachineCls, StateMachine)
@@ -564,7 +585,7 @@ class ModelWithStateMachine(BaseModelWithStateMachine):
     state_machine_name: str | None  # e.g. 'core.models.ArchiveResultMachine'
     state_field_name: str = "status"
     state_machine_attr: str = "sm"
-    bind_events_as_methods: bool = True
+    bind_events_as_methods: bool = False
 
     active_state = StatusChoices.STARTED
     retry_at_field_name: str = "retry_at"
@@ -601,6 +622,40 @@ class BaseStateMachine(StateMachine):
         setattr(self, self.model_attr_name, obj)
         super().__init__(obj, *args, **kwargs)
 
+    def _register_callbacks(self, listeners: list[object]):
+        """Register transition callbacks without scanning the Django model.
+
+        python-statemachine normally treats the wrapped model as a callback
+        listener.  That is useful when transition specs point at methods on the
+        domain object, but ArchiveBox keeps all transition guards/actions on the
+        machine classes themselves (`SnapshotMachine.can_start`,
+        `CrawlMachine.enter_sealed`, etc.).  Scanning the Django model therefore
+        only adds work: `dir(model)` is large, callback resolution walks that
+        attribute set for every state/transition, and the cost lands on every
+        `.sm` construction.
+
+        Keep support for explicit external listeners, but do not register
+        `self.model` as an implicit listener.  If a future machine wants model
+        methods as callbacks, pass that model explicitly as a listener at the
+        callsite so the cost is local and visible.
+        """
+        self._listeners.update({id(listener): listener for listener in listeners})
+        callbacks = Listeners.from_listeners(
+            (
+                Listener.from_obj(self, skip_attrs=self._protected_attrs),
+                *(Listener.from_obj(listener) for listener in listeners),
+            ),
+        )
+        registry = self._callbacks
+        callbacks.resolve(self._specs, registry=registry, allowed_references=SPECS_ALL)
+
+        check_callbacks = self._callbacks.check
+        for visited in iterate_states_and_transitions(self.states):
+            callbacks.resolve(visited._specs, registry=registry, allowed_references=SPECS_ALL)
+            check_callbacks(visited._specs)
+
+        self._callbacks.async_or_sync()
+
     def __repr__(self) -> str:
         obj = getattr(self, self.model_attr_name)
         return f"{self.__class__.__name__}[{obj.id}]"