Bump release version to 0.9.35rc467

Remove unused Python helpers and simplify equivalent test code without
changing active CLI, REST, model, runner, or UI behavior. This is the
mechanical first part of the simplification split; #1879 contains the
architectural and behavior-sensitive remainder. **Merge this PR first.**

- Delete unused hashing helpers, JSONL file loading, legacy TOML/INI
conversion helpers, and plugin-model discovery. Caller searches found no
active consumers; retained production definitions are AST-identical to
`dev`.
- Remove redundant result aliases, share identical POST payloads, and
parameterize equivalent test pairs while retaining their cases.
- Address Cubic's reviews: parse CSV and assert the exact header, URL
values, and sorted order; qualify the TLSNotary result lookup by its
full snapshot/plugin/hook identity.
- Correct the all-plugins HTTP fixture's TLSNotary expectation: every
snapshot must produce the specific HTTPS-only refusal, exit 1, and no
captured output. Other plugin statuses, output files, crawl topology,
and concurrent execution assertions remain enforced.
- Complete real parser dependency installation before public-add tests
deliberately stop the server. This prevents interrupting virtualenv
creation while preserving the crawl restart and metadata assertions.

Based on `dev` at `1fa2d4dc9`. **550 net lines removed**, excluding
`old/` and `docs/`; neither directory changes here. Runtime routing,
models, schemas, UI, dependencies, and architecture remain in #1879.

Validation:

- All-files `uv run --no-sync prek run --all-files` passes.
- The initial equivalent cleanup passed AST checks and 85 targeted
tests. Review follow-ups additionally pass both exact CSV tests, both
affected public-add runtime tests with a fresh dependency directory and
`--no-cache --no-sources`, and the full real depth-two all-plugins
crawl.
- [Full CI at
`42ed15b73`](https://github.com/ArchiveBox/ArchiveBox/actions/runs/35300660816)
**passed**, including every required lane. The subsequent one-line
TLSNotary lookup correction passed the full real all-plugins crawl
locally (131 seconds). All Cubic review threads are resolved.
- [Fresh CI at
`50a462e78`](https://github.com/ArchiveBox/ArchiveBox/actions/runs/35304078254)
is running.

Dockerfile

@@ -10,10 +10,10 @@
 #       --build-context abx-plugins=../abx-plugins \
 #       -t archivebox/abx-dl:dev
 #   docker buildx build . -f Dockerfile \
-#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.282 \
+#       --build-arg ABX_DL_IMAGE=archivebox/abx-dl:1.12.285 \
 #       -t archivebox:multistage
 
-ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.282
+ARG ABX_DL_IMAGE=archivebox/abx-dl:1.12.285
 
 FROM archivebox/sonic:1.4.9 AS sonic
 FROM ${ABX_DL_IMAGE} AS archivebox-runtime-base

archivebox/misc/hashing.py

@@ -1,260 +0,0 @@
-# Bootable hashing helpers (file content + mtime/size cache, mime detection).
-# MUST NOT import archivebox.config, archivebox.core, or Django — stdlib only.
-
-import hashlib
-import mimetypes
-from functools import lru_cache
-from pathlib import Path
-from collections.abc import Callable
-from datetime import datetime
-
-
-@lru_cache(maxsize=1024)
-def _cached_file_hash(filepath: str, size: int, mtime: float) -> str:
-    """Internal function to calculate file hash with cache key based on path, size and mtime."""
-    sha256_hash = hashlib.sha256()
-
-    with open(filepath, "rb") as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            sha256_hash.update(chunk)
-
-    return sha256_hash.hexdigest()
-
-
-@lru_cache(maxsize=10)
-def hash_file(file_path: Path, pwd: Path | None = None) -> str:
-    """Calculate SHA256 hash of a file with caching based on path, size and mtime."""
-    pwd = Path(pwd) if pwd else None
-    file_path = Path(file_path)
-    if not file_path.is_absolute():
-        file_path = pwd / file_path if pwd else file_path.absolute()
-
-    abs_path = file_path.resolve()
-    stat_info = abs_path.stat()
-
-    return _cached_file_hash(
-        str(abs_path),
-        stat_info.st_size,
-        stat_info.st_mtime,
-    )
-
-
-@lru_cache(maxsize=10)
-def get_dir_hashes(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict[str, str]:
-    """Calculate SHA256 hashes for all files and directories recursively."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    if not dir_path.is_dir():
-        raise ValueError(f"Not a directory: {dir_path}")
-    if max_depth < -1:
-        raise ValueError(f"max_depth must be >= -1, got {max_depth}")
-
-    # Get all files recursively
-    all_files = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=True,
-        include_dirs=False,
-        filter_func=filter_func,
-    )
-
-    hashes: dict[str, str] = {}
-    hashable_summary = []
-
-    # Calculate hashes for all files
-    for subfile in all_files:
-        subfile_path = dir_path / subfile
-        sha256_hash = hash_file(subfile_path)
-        hashes[subfile] = sha256_hash
-        hashable_summary.append(f"{sha256_hash}  ./{subfile}")
-
-    # Calculate hashes for all directories
-    subdirs = get_dir_entries(
-        dir_path,
-        pwd=pwd,
-        recursive=True,
-        include_files=False,
-        include_dirs=True,
-        include_hidden=False,
-        filter_func=filter_func,
-        max_depth=max_depth,
-    )
-
-    for subdir in subdirs:
-        subdir_path = dir_path / subdir
-        subdir_hashes = get_dir_hashes(
-            subdir_path,
-            filter_func=filter_func,
-            max_depth=0,
-        )
-        hashes[subdir] = subdir_hashes["."]
-
-    # Filter results by max_depth
-    if max_depth >= 0:
-        hashes = {path: value for path, value in hashes.items() if len(Path(path).parts) <= max_depth + 1}
-
-    # Calculate root directory hash
-    hashable_summary.sort()
-    root_sha256 = hashlib.sha256("\n".join(hashable_summary).encode()).hexdigest()
-    hashes["."] = root_sha256
-
-    return hashes
-
-
-@lru_cache(maxsize=128)
-def get_dir_entries(
-    dir_path: Path,
-    pwd: Path | None = None,
-    recursive: bool = True,
-    include_files: bool = True,
-    include_dirs: bool = True,
-    include_hidden: bool = False,
-    filter_func: Callable | None = None,
-    max_depth: int = -1,
-) -> tuple[str, ...]:
-    """Get filtered list of directory entries."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    results = []
-
-    def process_path(path: Path, depth: int):
-        if not include_hidden and path.name.startswith("."):
-            return False
-        if max_depth >= 0 and depth > max_depth:
-            return False
-        if filter_func:
-            info = {
-                "abspath": str(path.absolute()),
-                "relpath": str(path.relative_to(dir_path)),
-            }
-            if not filter_func(info):
-                return False
-        return True
-
-    for path in dir_path.rglob("*") if recursive else dir_path.glob("*"):
-        current_depth = len(path.relative_to(dir_path).parts)
-
-        if path.is_file() and include_files and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-        elif path.is_dir() and include_dirs and process_path(path, current_depth):
-            results.append(str(path.relative_to(dir_path)))
-
-        if not recursive:
-            break
-
-    return tuple(sorted(results))  # Make immutable for caching
-
-
-@lru_cache(maxsize=1024)
-def get_dir_sizes(dir_path: Path, pwd: Path | None = None, **kwargs) -> dict[str, int]:
-    """Calculate sizes for all files and directories recursively."""
-    sizes: dict[str, int] = {}
-    hashes = get_dir_hashes(dir_path, pwd=pwd, **kwargs)
-    dir_path = Path(dir_path)
-
-    for path_key in hashes:
-        full_path = dir_path / path_key
-        if full_path.is_file():
-            sizes[path_key] = full_path.stat().st_size
-        else:
-            total = 0
-            for file_path in full_path.rglob("*"):
-                if file_path.is_file() and not file_path.name.startswith("."):
-                    total += file_path.stat().st_size
-            sizes[path_key + "/"] = total
-
-    return sizes
-
-
-@lru_cache(maxsize=10)
-def get_dir_info(dir_path: Path, pwd: Path | None = None, filter_func: Callable | None = None, max_depth: int = -1) -> dict:
-    """Get detailed information about directory contents including hashes and sizes."""
-    pwd = Path(pwd) if pwd else None
-    dir_path = Path(dir_path)
-    if not dir_path.is_absolute():
-        dir_path = pwd / dir_path if pwd else dir_path.absolute()
-
-    hashes = get_dir_hashes(dir_path, pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-    sizes = get_dir_sizes(str(dir_path), pwd=pwd, filter_func=filter_func, max_depth=max_depth)
-
-    num_total_subpaths = sum(1 for name in hashes if name != ".")
-    details = {}
-
-    for filename, sha256_hash in sorted(hashes.items()):
-        abs_path = (dir_path / filename).resolve()
-        stat_info = abs_path.stat()
-        num_subpaths = sum(1 for p in hashes if p.startswith(filename + "/"))
-        is_dir = abs_path.is_dir()
-        if is_dir:
-            mime_type = "inode/directory"
-            basename = abs_path.name
-            extension = ""
-            num_bytes = sizes[filename + "/"]
-            if filename == ".":
-                num_subpaths = num_total_subpaths
-            else:
-                filename += "/"
-                num_subpaths = num_subpaths
-        else:  # is_file
-            num_subpaths = None
-            mime_type = mimetypes.guess_type(str(abs_path))[0]
-            extension = abs_path.suffix
-            basename = abs_path.name.rsplit(extension, 1)[0]
-            num_bytes = sizes[filename]
-
-        details[filename] = {
-            "basename": basename,
-            "mime_type": mime_type,
-            "extension": extension,
-            "num_subpaths": num_subpaths,
-            "num_bytes": num_bytes,
-            "hash_sha256": sha256_hash,
-            "created_at": datetime.fromtimestamp(stat_info.st_ctime).isoformat(),
-            "modified_at": datetime.fromtimestamp(stat_info.st_mtime).isoformat(),
-        }
-
-        if filter_func and not filter_func(details[filename]):
-            del details[filename]
-
-    return details
-
-
-if __name__ == "__main__":
-    import json
-
-    dir_info = get_dir_info(Path("."), max_depth=6)
-    with open(".hashes.json", "w") as f:
-        json.dump(dir_info, f, indent=4)
-    print("Wrote .hashes.json")
-
-# Example output:
-# {
-#     ".": {
-#         "basename": "misc",
-#         "mime_type": "inode/directory",
-#         "extension": "",
-#         "num_subpaths": 25,
-#         "num_bytes": 214677,
-#         "hash_sha256": "addfacf88b2ff6b564846415fb7b21dcb7e63ee4e911bc0aec255ee354958530",
-#         "created_at": "2024-12-04T00:08:38.537449",
-#         "modified_at": "2024-12-04T00:08:38.537449"
-#     },
-#     "__init__.py": {
-#         "basename": "__init__",
-#         "mime_type": "text/x-python",
-#         "extension": ".py",
-#         "num_subpaths": null,
-#         "num_bytes": 32,
-#         "hash_sha256": "b0e5e7ff17db3b60535cf664282787767c336e3e203a43e21b6326c6fe457551",
-#         "created_at": "2024-10-08T00:51:41.001359",
-#         "modified_at": "2024-10-08T00:51:41.001359"
-#     },
-#     ...
-# }

archivebox/misc/jsonl.py

@@ -32,7 +32,6 @@ import sys
 import json
 from typing import Any, TextIO
 from collections.abc import Iterable, Iterator
-from pathlib import Path
 
 
 # Type constants for JSONL records
@@ -120,19 +119,6 @@ def read_stdin(stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
             yield record
 
 
-def read_file(path: Path) -> Iterator[dict[str, Any]]:
-    """
-    Read JSONL or plain URLs from a file.
-
-    Yields parsed records as dicts.
-    """
-    with open(path) as f:
-        for line in f:
-            record = parse_line(line)
-            if record:
-                yield record
-
-
 def read_args_or_stdin(args: Iterable[str], stream: TextIO | None = None) -> Iterator[dict[str, Any]]:
     """
     Read from CLI arguments if provided, otherwise from stdin.

archivebox/misc/toml_util.py

@@ -2,94 +2,11 @@ from typing import Any, cast
 from collections.abc import Callable
 
 import json
-import ast
-import inspect
 import toml
 import re
-import configparser
 
 from pathlib import Path, PosixPath
 
-from pydantic.json_schema import GenerateJsonSchema
-from pydantic_core import to_jsonable_python
-
-JSONValue = str | bool | int | None | list["JSONValue"]
-
-TOML_HEADER = "# Converted from INI to TOML format: https://toml.io/en/\n\n"
-
-
-def load_ini_value(val: str) -> JSONValue:
-    """Convert lax INI values into strict TOML-compliant (JSON) values"""
-    if val.lower() in ("true", "yes", "1"):
-        return True
-    if val.lower() in ("false", "no", "0"):
-        return False
-    if val.isdigit():
-        return int(val)
-
-    try:
-        return ast.literal_eval(val)
-    except Exception:
-        pass
-
-    try:
-        return json.loads(val)
-    except Exception:
-        pass
-
-    return val
-
-
-def convert(ini_str: str) -> str:
-    """Convert a string of INI config into its TOML equivalent (warning: strips comments)"""
-
-    config = configparser.ConfigParser()
-    setattr(config, "optionxform", str)  # capitalize key names
-    config.read_string(ini_str)
-
-    # Initialize an empty dictionary to store the TOML representation
-    toml_dict = {}
-
-    # Iterate over each section in the INI configuration
-    for section in config.sections():
-        toml_dict[section] = {}
-
-        # Iterate over each key-value pair in the section
-        for key, value in config.items(section):
-            parsed_value = load_ini_value(value)
-
-            # Convert the parsed value to its TOML-compatible JSON representation
-            toml_dict[section.upper()][key.upper()] = json.dumps(parsed_value)
-
-    # Build the TOML string
-    toml_str = TOML_HEADER
-    for section, items in toml_dict.items():
-        toml_str += f"[{section}]\n"
-        for key, value in items.items():
-            toml_str += f"{key} = {value}\n"
-        toml_str += "\n"
-
-    return toml_str.strip()
-
-
-class JSONSchemaWithLambdas(GenerateJsonSchema):
-    """
-    Encode lambda functions in default values properly.
-    Usage:
-    >>> json.dumps(value, encoder=JSONSchemaWithLambdas())
-    """
-
-    def encode_default(self, dft: Any) -> Any:
-        config = self._config
-        if isinstance(dft, Callable):
-            return "{{lambda " + inspect.getsource(dft).split("=lambda ")[-1].strip()[:-1] + "}}"
-        return to_jsonable_python(
-            dft,
-            timedelta_mode=config.ser_json_timedelta,
-            bytes_mode=config.ser_json_bytes,
-            serialize_unknown=True,
-        )
-
 
 def better_toml_dump_str(val: Any) -> str:
     try:

archivebox/plugins/discovery.py

@@ -55,10 +55,6 @@ def get_plugins() -> list[str]:
     return sorted(get_plugin_catalog())
 
 
-def get_plugin_models():
-    return get_plugin_catalog().plugins
-
-
 def get_plugin_name(plugin: str) -> str:
     """
     Get the base plugin name without numeric prefix.

archivebox/tests/test_auth_ldap.py

@@ -153,7 +153,7 @@ class TestArchiveBoxWithLDAP:
 
     def test_archivebox_init_without_ldap(self, tmp_path):
         """Test that archivebox init works without LDAP enabled."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["init"],
             cwd=tmp_path,
             timeout=45,
@@ -161,14 +161,13 @@ class TestArchiveBoxWithLDAP:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
-        assert code == 0, f"archivebox init failed: {stderr}"
+        assert result.returncode == 0, f"archivebox init failed: {result.stderr}"
 
     def test_archivebox_version_with_ldap_config(self, tmp_path):
         """Test that archivebox version works with LDAP config set."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["version"],
             cwd=tmp_path,
             timeout=10,
@@ -179,10 +178,9 @@ class TestArchiveBoxWithLDAP:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should succeed
-        assert code == 0, f"archivebox version failed: {stderr}"
+        assert result.returncode == 0, f"archivebox version failed: {result.stderr}"
 
 
 class TestLDAPConfigValidationInArchiveBox:
@@ -190,7 +188,7 @@ class TestLDAPConfigValidationInArchiveBox:
 
     def test_archivebox_init_with_incomplete_ldap_config(self, tmp_path):
         """Test that archivebox init fails with helpful error when LDAP config is incomplete."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["init"],
             cwd=tmp_path,
             timeout=45,
@@ -201,10 +199,9 @@ class TestLDAPConfigValidationInArchiveBox:
             default_cli_env=True,
             disable_extractors=True,
         )
-        _, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
         # Should fail with validation error
-        assert code != 0, "Should fail with incomplete LDAP config"
+        assert result.returncode != 0, "Should fail with incomplete LDAP config"
 
         # Check error message
-        assert "LDAP_* config options must all be set" in stderr, f"Expected validation error message in: {stderr}"
+        assert "LDAP_* config options must all be set" in result.stderr, f"Expected validation error message in: {result.stderr}"

archivebox/tests/test_cli_add.py

@@ -1204,7 +1204,7 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive,
     root_url = recursive_test_site["root_url"]
     child_url = recursive_test_site["child_urls"][0]
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         [
             "add",
             "--depth=2",
@@ -1219,8 +1219,7 @@ def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive,
         env=env,
         timeout=180,
     )
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-    assert returncode == 0, stderr or stdout
+    assert result.returncode == 0, result.stderr or result.stdout
 
     with use_archivebox_db(initialized_archive):
         crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()

archivebox/tests/test_cli_archiveresult.py

@@ -142,18 +142,17 @@ class TestArchiveResultCreate:
         assert crawl_result.returncode == 0, crawl_result.stderr
         crawl_record = parse_jsonl_output(crawl_result.stdout)[0]
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["archiveresult", "create"],
             stdin=crawl_result.stdout,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Passed through" in stderr
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        assert "Passed through" in result.stderr
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["id"] == crawl_record["id"]
 
@@ -163,16 +162,15 @@ class TestArchiveResultList:
 
     def test_list_empty(self, initialized_archive):
         """List with no archive results returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["archiveresult", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 archive results" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 archive results" in result.stderr
 
     def test_list_filter_by_status(self, initialized_archive):
         """Filter archive results by status."""

archivebox/tests/test_cli_crawl.py

@@ -33,19 +33,18 @@ class TestCrawlCreate:
         """Create crawl from URL arguments."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
-        assert "Created crawl" in stderr
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert "Created crawl" in result.stderr
 
         # Check JSONL output
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["type"] == "Crawl"
         assert url in records[0]["urls"]
@@ -55,18 +54,17 @@ class TestCrawlCreate:
         urls = [create_test_url() for _ in range(3)]
         stdin = "\n".join(urls)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
 
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         crawl = records[0]
         assert crawl["type"] == "Crawl"
@@ -78,32 +76,30 @@ class TestCrawlCreate:
         """Create crawl with --depth flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", "--depth=2", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert records[0]["max_depth"] == 2
 
     def test_create_with_tag(self, initialized_archive):
         """Create crawl with --tag flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create", "--tag=test-tag", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert "test-tag" in records[0].get("tags_str", "")
 
     def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
@@ -119,17 +115,16 @@ class TestCrawlCreate:
         url = create_test_url()
         stdin = tag_result.stdout + json.dumps({"url": url}) + "\n"
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
 
         assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
         assert any(record.get("type") == "Crawl" and url in record["urls"] for record in records)
@@ -163,32 +158,30 @@ class TestCrawlList:
 
     def test_list_empty(self, initialized_archive):
         """List with no crawls returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 crawls" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 crawls" in result.stderr
 
     def test_list_returns_created(self, initialized_archive):
         """List returns previously created crawls."""
         url = create_test_url()
         run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) >= 1
         assert any(url in r.get("urls", "") for r in records)
 
@@ -197,16 +190,15 @@ class TestCrawlList:
         url = create_test_url()
         run_archivebox_cmd(["crawl", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list", "--status=queued"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         for r in records:
             assert r["status"] == "queued"
 
@@ -221,16 +213,15 @@ class TestCrawlList:
                 disable_extractors=True,
             )
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["crawl", "list", "--limit=2"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 2
 
 

archivebox/tests/test_cli_list.py

@@ -3,6 +3,8 @@ Tests for archivebox list command.
 Verify list emits snapshot JSONL and applies the documented filters.
 """
 
+import csv
+import io
 import json
 from pathlib import Path
 
@@ -427,16 +429,15 @@ def test_snapshot_list_search_meta(initialized_archive):
     url = create_test_url(domain="meta-search-example.com")
     run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["snapshot", "list", "--search=meta", "meta-search-example.com"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, f"Command failed: {stderr}"
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, f"Command failed: {result.stderr}"
+    records = parse_jsonl_output(result.stdout)
     assert len(records) == 1
     assert "meta-search-example.com" in records[0]["url"]
 
@@ -446,16 +447,15 @@ def test_list_search_meta_matches_metadata(initialized_archive):
     url = create_test_url(domain="top-level-meta-search-example.com")
     run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["list", "--search=meta", "top-level-meta-search-example.com"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, f"Command failed: {stderr}"
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, f"Command failed: {result.stderr}"
+    records = parse_jsonl_output(result.stdout)
     assert len(records) == 1
     assert "top-level-meta-search-example.com" in records[0]["url"]
 
@@ -468,11 +468,10 @@ def test_search_command_finds_snapshots(initialized_archive):
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(["search", "example"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "example"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code == 0, stderr
-    assert "example" in stdout
+    assert result.returncode == 0, result.stderr
+    assert "example" in result.stdout
 
 
 def test_search_command_returns_no_results_for_missing_term(initialized_archive):
@@ -483,22 +482,20 @@ def test_search_command_returns_no_results_for_missing_term(initialized_archive)
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["search", "nonexistentterm12345"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code in [0, 1]
+    assert result.returncode in [0, 1]
 
 
 def test_search_command_on_empty_archive(initialized_archive):
-    _cmd_result = run_archivebox_cmd(["search", "anything"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "anything"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code in [0, 1]
+    assert result.returncode in [0, 1]
 
 
 def test_search_command_outputs_matching_snapshots_as_jsonl(initialized_archive):
@@ -509,11 +506,10 @@ def test_search_command_outputs_matching_snapshots_as_jsonl(initialized_archive)
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(["search"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code == 0, stderr
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, result.stderr
+    records = parse_jsonl_output(result.stdout)
     assert any("example.com" in row.get("url", "") for row in records)
 
 
@@ -576,52 +572,47 @@ def test_search_command_csv_outputs_requested_column(initialized_archive):
         disable_extractors=True,
     )
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["search", "--csv", "url", "--with-headers"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, stderr
-    assert "url" in stdout
-    assert "example.com" in stdout
+    assert result.returncode == 0, result.stderr
+    assert list(csv.reader(io.StringIO(result.stdout))) == [["url"], ["https://example.com"]]
 
 
 def test_search_command_with_headers_requires_structured_output_format(initialized_archive):
-    _cmd_result = run_archivebox_cmd(["search", "--with-headers"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "--with-headers"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code != 0
-    assert "requires" in stderr.lower()
-    assert "json" in stderr.lower()
+    assert result.returncode != 0
+    assert "requires" in result.stderr.lower()
+    assert "json" in result.stderr.lower()
 
 
 def test_search_command_sort_option_runs_successfully(initialized_archive):
     for url in ["https://iana.org", "https://example.com"]:
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["search", "--csv", "url", "--sort=url"],
         cwd=initialized_archive,
         default_cli_env=True,
         disable_extractors=True,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-    assert code == 0, stderr
-    assert "example.com" in stdout or "iana.org" in stdout
+    assert result.returncode == 0, result.stderr
+    assert list(csv.reader(io.StringIO(result.stdout))) == [["https://example.com"], ["https://iana.org"]]
 
 
 def test_search_command_help_lists_supported_filters(initialized_archive):
-    _cmd_result = run_archivebox_cmd(["search", "--help"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
-    stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
+    result = run_archivebox_cmd(["search", "--help"], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-    assert code == 0
-    assert "--url__icontains" in stdout
-    assert "--crawl-id" in stdout
-    assert "--status" in stdout
-    assert "--sort" in stdout
-    assert "--json" in stdout
-    assert "--html" in stdout
+    assert result.returncode == 0
+    assert "--url__icontains" in result.stdout
+    assert "--crawl-id" in result.stdout
+    assert "--status" in result.stdout
+    assert "--sort" in result.stdout
+    assert "--json" in result.stdout
+    assert "--html" in result.stdout

archivebox/tests/test_cli_run.py

@@ -53,15 +53,14 @@ def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archiv
     env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false")
     _install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
 
-    _cmd_result = run_archivebox_cmd(
+    result = run_archivebox_cmd(
         ["snapshot", "create", recursive_test_site["root_url"]],
         cwd=initialized_archive,
         env=env,
         timeout=60,
     )
-    stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-    assert returncode == 0, stderr or stdout
-    records = parse_jsonl_output(stdout)
+    assert result.returncode == 0, result.stderr or result.stdout
+    records = parse_jsonl_output(result.stdout)
     snapshot_id = next(record["id"] for record in records if record.get("type") == "Snapshot")
     with use_archivebox_db(initialized_archive):
         browser_state = Snapshot.objects.get(id=snapshot_id).output_dir / "chrome" / "browser.json"
@@ -107,7 +106,7 @@ class TestRunWithCrawl:
         )
         assert create_result.returncode == 0, create_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=create_result.stdout,
             cwd=initialized_archive,
@@ -116,12 +115,11 @@ class TestRunWithCrawl:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
 
         # Should output the created Crawl
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         crawl_records = [r for r in records if r.get("type") == "Crawl"]
         assert len(crawl_records) >= 1
         assert crawl_records[0].get("id")  # Should have an id now
@@ -170,7 +168,7 @@ class TestRunWithSnapshot:
         )
         assert create_result.returncode == 0, create_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=create_result.stdout,
             cwd=initialized_archive,
@@ -179,11 +177,10 @@ class TestRunWithSnapshot:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
 
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         snapshot_records = [r for r in records if r.get("type") == "Snapshot"]
         assert len(snapshot_records) >= 1
         assert snapshot_records[0].get("id")
@@ -220,7 +217,7 @@ class TestRunWithSnapshot:
     def test_run_with_plain_url(self, initialized_archive):
         """Run accepts plain URL records (no type field)."""
         url = create_test_url()
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=url + "\n",
             cwd=initialized_archive,
@@ -229,10 +226,9 @@ class TestRunWithSnapshot:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) >= 1
 
 
@@ -405,7 +401,7 @@ class TestRunRecovery:
             crawl_id = crawl.id
             snapshot_id = snapshot.id
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run", "--maintenance-only"],
             cwd=initialized_archive,
             timeout=90,
@@ -413,13 +409,12 @@ class TestRunRecovery:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, stdout + stderr
-        assert "Repairing" in stderr
-        assert "Resuming 1 Crawl(s) with pending URLs ready to archive" in stderr
-        assert "interrupted before" in stderr
-        assert "remaining URLs" in stderr
+        assert result.returncode == 0, result.stdout + result.stderr
+        assert "Repairing" in result.stderr
+        assert "Resuming 1 Crawl(s) with pending URLs ready to archive" in result.stderr
+        assert "interrupted before" in result.stderr
+        assert "remaining URLs" in result.stderr
 
         with use_archivebox_db(initialized_archive):
             crawl = Crawl.objects.get(id=crawl_id)
@@ -444,17 +439,16 @@ class TestRunPassThrough:
         assert tag_result.returncode == 0, tag_result.stderr
         tag_record = parse_jsonl_output(tag_result.stdout)[0]
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=tag_result.stdout,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         tag_records = [record for record in records if record.get("type") == "Tag"]
         assert len(tag_records) == 1
         assert tag_records[0]["id"] == tag_record["id"]
@@ -471,7 +465,7 @@ class TestRunPassThrough:
         )
         assert create_result.returncode == 0, create_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=create_result.stdout,
             cwd=initialized_archive,
@@ -480,10 +474,9 @@ class TestRunPassThrough:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         # Should have at least the Crawl in output
         assert len(records) >= 1
 
@@ -516,7 +509,7 @@ class TestRunMixedInput:
         )
         assert snapshot_result.returncode == 0, snapshot_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=tag_result.stdout + snapshot_result.stdout,
             cwd=initialized_archive,
@@ -525,10 +518,9 @@ class TestRunMixedInput:
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
 
         types = {record.get("type") for record in records}
         assert {"Crawl", "Snapshot", "Tag"}.issubset(types)
@@ -539,16 +531,15 @@ class TestRunEmpty:
 
     def test_run_empty_stdin(self, initialized_archive):
         """Run with empty stdin returns success."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin="",
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
+        assert result.returncode == 0
 
     def test_run_no_runnable_records_to_process(self, initialized_archive):
         """Run with only a real non-runnable Tag reports no work."""
@@ -560,17 +551,16 @@ class TestRunEmpty:
         )
         assert tag_result.returncode == 0, tag_result.stderr
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["run"],
             stdin=tag_result.stdout,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "No records to process" in stderr
+        assert result.returncode == 0
+        assert "No records to process" in result.stderr
 
 
 class TestRunDaemonMode:

archivebox/tests/test_cli_server.py

@@ -124,10 +124,17 @@ def test_server_auth_secret_and_cookie_settings_are_restart_stable(tmp_path):
     assert first_lines[3:] == ["None", "False", "False"]
 
 
-def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path):
+@pytest.mark.parametrize(
+    "base_url, secure",
+    [
+        ("https://archive.example.com", True),
+        ("", False),
+    ],
+)
+def test_base_url_controls_secure_cookies(tmp_path, base_url, secure):
     (tmp_path / ".archivebox_id").write_text("testcoll")
     env = os.environ.copy()
-    env["BASE_URL"] = "https://archive.example.com"
+    env["BASE_URL"] = base_url
     env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
     repo_root = Path(__file__).resolve().parents[2]
     env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}"
@@ -155,45 +162,8 @@ def test_https_base_url_enables_proxy_ssl_header_and_secure_cookies(tmp_path):
     )
 
     assert json.loads(result.stdout) == {
-        "csrf_secure": True,
-        "session_secure": True,
-        "proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"],
-    }
-
-
-def test_unconfigured_base_url_enables_proxy_ssl_header_without_secure_cookies(tmp_path):
-    (tmp_path / ".archivebox_id").write_text("testcoll")
-    env = os.environ.copy()
-    env["BASE_URL"] = ""
-    env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
-    repo_root = Path(__file__).resolve().parents[2]
-    env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{env.get('PYTHONPATH', '')}"
-
-    result = subprocess.run(
-        [
-            sys.executable,
-            "-c",
-            (
-                "import django, json;"
-                "django.setup();"
-                "from django.conf import settings;"
-                "print(json.dumps({"
-                "'csrf_secure': settings.CSRF_COOKIE_SECURE,"
-                "'session_secure': settings.SESSION_COOKIE_SECURE,"
-                "'proxy_ssl_header': settings.SECURE_PROXY_SSL_HEADER,"
-                "}))"
-            ),
-        ],
-        capture_output=True,
-        text=True,
-        check=True,
-        env=env,
-        cwd=tmp_path,
-    )
-
-    assert json.loads(result.stdout) == {
-        "csrf_secure": False,
-        "session_secure": False,
+        "csrf_secure": secure,
+        "session_secure": secure,
         "proxy_ssl_header": ["HTTP_X_FORWARDED_PROTO", "https"],
     }
 
@@ -759,9 +729,8 @@ def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(initi
 
         resumed = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}-resumed.log", env=env)
         resumed_log = resumed.log_path
-        _cmd_result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env, timeout=60)
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, stderr or stdout
+        result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env, timeout=60)
+        assert result.returncode == 0, result.stderr or result.stdout
 
         os.kill(resumed.pid, signal.SIGTERM)
         resumed.wait(timeout=20)
@@ -783,14 +752,13 @@ def test_live_daemonized_server_keeps_supervisord_owned_by_archivebox_parent(ini
     port = get_free_port()
     bind_url = f"http://127.0.0.1:{port}"
     try:
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["server", "--daemonize", f"127.0.0.1:{port}"],
             cwd=initialized_archive,
             env=env,
             timeout=90,
         )
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, stderr or stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         _wait_for_archivebox_workers(initialized_archive, env, ("worker_daphne", "worker_runner"), timeout=30)
         assert_port_open("127.0.0.1", port, timeout=30)
 

archivebox/tests/test_cli_snapshot.py

@@ -33,18 +33,17 @@ class TestSnapshotCreate:
         """Create snapshot from URL arguments."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
-        assert "Created" in stderr
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        assert "Created" in result.stderr
 
-        records = parse_jsonl_output(stdout)
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert records[0]["type"] == "Snapshot"
         assert records[0]["url"] == url
@@ -83,16 +82,15 @@ class TestSnapshotCreate:
         """Create snapshot with --tag flag."""
         url = create_test_url()
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create", "--tag=test-tag", url],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert "test-tag" in records[0].get("tags", "")
 
     def test_create_passes_through_tag_emitted_by_cli(self, initialized_archive):
@@ -108,17 +106,16 @@ class TestSnapshotCreate:
         url = create_test_url()
         stdin = tag_result.stdout + url + "\n"
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create"],
             stdin=stdin,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
 
         assert any(record.get("type") == "Tag" and record["id"] == tag_record["id"] for record in records)
         assert any(record.get("type") == "Snapshot" and record["url"] == url for record in records)
@@ -127,16 +124,15 @@ class TestSnapshotCreate:
         """Create snapshots from multiple URLs."""
         urls = [create_test_url() for _ in range(3)]
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "create"] + urls,
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 3
 
         created_urls = {r["url"] for r in records}
@@ -149,32 +145,30 @@ class TestSnapshotList:
 
     def test_list_empty(self, initialized_archive):
         """List with no snapshots returns empty."""
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        _stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        assert "Listed 0 snapshots" in stderr
+        assert result.returncode == 0
+        assert "Listed 0 snapshots" in result.stderr
 
     def test_list_returns_created(self, initialized_archive):
         """List returns previously created snapshots."""
         url = create_test_url()
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) >= 1
         assert any(r.get("url") == url for r in records)
 
@@ -183,16 +177,15 @@ class TestSnapshotList:
         url = create_test_url()
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--status=queued"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         for r in records:
             assert r["status"] == "queued"
 
@@ -201,16 +194,15 @@ class TestSnapshotList:
         url = create_test_url(domain="unique-domain-12345.com")
         run_archivebox_cmd(["snapshot", "create", url], cwd=initialized_archive, default_cli_env=True, disable_extractors=True)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--url__icontains=unique-domain-12345"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 1
         assert "unique-domain-12345" in records[0]["url"]
 
@@ -224,16 +216,15 @@ class TestSnapshotList:
                 disable_extractors=True,
             )
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--limit=2"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 2
 
     def test_list_with_sort_and_limit(self, initialized_archive):
@@ -246,16 +237,15 @@ class TestSnapshotList:
                 disable_extractors=True,
             )
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["snapshot", "list", "--limit=2", "--sort=-created_at"],
             cwd=initialized_archive,
             default_cli_env=True,
             disable_extractors=True,
         )
-        stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert code == 0, f"Command failed: {stderr}"
-        records = parse_jsonl_output(stdout)
+        assert result.returncode == 0, f"Command failed: {result.stderr}"
+        records = parse_jsonl_output(result.stdout)
         assert len(records) == 2
 
 

archivebox/tests/test_recursive_crawl.py

@@ -635,7 +635,22 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
         for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
         if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
     ]
-    assert not unexpected_results
+    # This fixture serves HTTP. TLSNotary must reject every plaintext document;
+    # a setup, timeout, or verifier error must not count as that expected refusal.
+    tlsnotary_results = [row for row in archive_results if row[3] == "tlsnotary"]
+    assert {row[1] for row in tlsnotary_results} == expected_urls
+    assert len(tlsnotary_results) == len(expected_urls)
+    with use_archivebox_db(initialized_archive):
+        for snapshot_id, _url, _depth, _plugin, hook_name, status, files, size, _output in tlsnotary_results:
+            assert status == ArchiveResult.StatusChoices.FAILED
+            assert not files
+            assert size == 0
+            process = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="tlsnotary", hook_name=hook_name).process
+            assert process is not None
+            assert process.status == Process.StatusChoices.EXITED
+            assert process.exit_code == 1
+            assert process.stderr.strip() == "[tlsnotary] TLSNotary requires an HTTPS document"
+    assert not [result for result in unexpected_results if result["plugin"] != "tlsnotary"]
     ytdlp_results = [
         (url, status, output_str)
         for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
@@ -682,19 +697,6 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
     assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
 
     assert processes
-    failed_hook_results = [
-        {
-            "url": url,
-            "depth": depth,
-            "plugin": plugin,
-            "hook_name": hook_name,
-            "status": status,
-            "output_str": output_str,
-        }
-        for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
-        if status == ArchiveResult.StatusChoices.FAILED and plugin != "archivedotorg"
-    ]
-    assert not failed_hook_results
     assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
 
     intervals = []

archivebox/tests/test_server_security_browser.py

@@ -963,14 +963,13 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
             log_name="archivewebpage_server.log",
         )
         get_http_response(port, host=f"archivebox.localhost:{port}", path="/")
-        _cmd_result = run_archivebox_cmd(
+        command_result = run_archivebox_cmd(
             ["add", "--depth=0", "--max-urls=1", "--plugins=archivewebpage", url],
             cwd=initialized_archive,
             env=env,
             timeout=120,
         )
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, f"archivebox add failed:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"
+        assert command_result.returncode == 0, f"archivebox add failed:\nSTDOUT:\n{command_result.stdout}\nSTDERR:\n{command_result.stderr}"
 
         capture = _get_archivewebpage_capture(initialized_archive, url)
         snapshot_host = f"{get_snapshot_subdomain(capture['snapshot_id'])}.archivebox.localhost:{port}"

archivebox/tests/test_snapshot_service.py

@@ -163,14 +163,15 @@ def test_snapshot_service_cli_add_seals_snapshot_and_writes_indexes(tmp_path, re
 
     port = get_free_port()
     env = cli_env(port=port, server=True, PLUGINS="wget", SAVE_WGET="True")
-    _cmd_result = run_archivebox_cmd(
+    command_result = run_archivebox_cmd(
         ["add", "--depth=0", "--plugins=wget", recursive_test_site["root_url"]],
         cwd=tmp_path,
         env=env,
         timeout=180,
     )
-    stdout, stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-    assert code == 0, f"archivebox add failed with code {code}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"
+    assert command_result.returncode == 0, (
+        f"archivebox add failed with code {command_result.returncode}\nSTDOUT:\n{command_result.stdout}\nSTDERR:\n{command_result.stderr}"
+    )
 
     state = _snapshot_state(tmp_path, recursive_test_site["root_url"])
     snapshot_dir = state["snapshot_dir"]

archivebox/tests/test_takeover_util.py

@@ -387,14 +387,13 @@ def test_live_second_server_takes_over_existing_server_process(tmp_path, initial
         assert "A newer archivebox process took over the orchestrator, server" in first_text
         assert "Starting orchestrator, server" in second_text
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["status"],
             cwd=tmp_path,
             env=env,
             timeout=60,
         )
-        stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert returncode == 0, stderr or stdout
+        assert result.returncode == 0, result.stderr or result.stdout
 
         first_resumes = first_log.read_text(encoding="utf-8", errors="replace").count("Other newer archivebox process")
         stop_archivebox_process(second, signal.SIGTERM)
@@ -422,15 +421,14 @@ def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, init
         supervisor_pid_before = supervisor_pid_from_log(server_log)
         daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne")
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["update", "--index-only", "--before=0"],
             cwd=tmp_path,
             env=env,
             timeout=90,
         )
-        update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert update_returncode == 0, update_stderr or update_stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         assert pid_is_alive(server.pid)
         assert pid_is_alive(supervisor_pid_before)
         assert pid_is_alive(daphne_pid_before)
@@ -471,15 +469,14 @@ def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tm
         runner_spawn_text = "spawned: 'worker_runner' with pid"
         runner_spawn_count = supervisord_log.read_text(encoding="utf-8", errors="replace").count(runner_spawn_text)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["update", "--index-only", "--batch-size=1"],
             cwd=tmp_path,
             env=env,
             timeout=180,
         )
-        update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
 
-        assert update_returncode == 0, update_stderr or update_stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         assert pid_is_alive(server.pid)
         assert supervisor_pid_from_log(server_log) == supervisor_pid_before
         assert pid_is_alive(daphne_pid_before)
@@ -628,14 +625,13 @@ def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_a
                 wait_for_pid_to_disappear(daphne_pids[index - 1], timeout=15)
                 wait_for_pid_to_disappear(runner_pids[index - 1], timeout=15)
 
-            _cmd_result = run_archivebox_cmd(
+            result = run_archivebox_cmd(
                 ["status"],
                 cwd=tmp_path,
                 env=env,
                 timeout=60,
             )
-            stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-            assert returncode == 0, stderr or stdout
+            assert result.returncode == 0, result.stderr or result.stdout
 
         assert pid_is_alive(servers[-1].pid)
         assert all(pid_is_alive(server.pid) for server in servers)
@@ -673,14 +669,13 @@ def test_live_background_add_survives_server_exit_and_foreground_run_reclaims(tm
         server_log = server.log_path
         supervisor_pid_before = supervisor_pid_from_log(server_log)
 
-        _cmd_result = run_archivebox_cmd(
+        result = run_archivebox_cmd(
             ["update", "--index-only", "--batch-size=10"],
             cwd=tmp_path,
             env=env,
             timeout=90,
         )
-        update_stdout, update_stderr, update_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
-        assert update_returncode == 0, update_stderr or update_stdout
+        assert result.returncode == 0, result.stderr or result.stdout
         assert pid_is_alive(server.pid)
         assert pid_is_alive(supervisor_pid_before)
         assert supervisor_pid_from_log(server_log) == supervisor_pid_before

archivebox/tests/test_ui_add_view.py

@@ -20,6 +20,23 @@ WEB_HOST = "web.archivebox.localhost:8000"
 ADMIN_HOST = "admin.archivebox.localhost:8000"
 
 
+ADD_FORM_DEFAULTS = {
+    "tag": "",
+    "depth": "0",
+    "max_urls": "0",
+    "crawl_max_size": "0",
+    "snapshot_max_size": "0",
+    "url_filters_allowlist": "",
+    "url_filters_denylist": "",
+    "notes": "",
+    "schedule": "",
+    "persona": "Default",
+    "permissions": "public",
+    "start_paused": "",
+    "config": "{}",
+}
+
+
 @pytest.fixture
 def admin_user(db):
     return User.objects.create_superuser(
@@ -140,22 +157,13 @@ def test_add_view_staff_user_cannot_override_raw_or_plugin_config(client):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/staff-config",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
             "crawl_timeout": "0",
             "timeout": "60",
-            "snapshot_max_size": "0",
             "delete_after": "0",
             "crawl_max_concurrent_snapshots": "1",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
-            "notes": "",
             "schedule": "daily",
-            "persona": "Default",
-            "permissions": "public",
             "start_paused": "on",
             "main_plugins": ["wget"],
             "plugin_config__wget__WGET_TIMEOUT": "77",
@@ -246,6 +254,7 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com\nhttps://cdn.example.com/asset.js",
             "tag": "alpha,beta",
             "depth": "1",
@@ -260,11 +269,6 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
             "url_filters_denylist": "cdn.example.com",
             "url_filters_only_new": "1",
             "notes": "Created from /add/",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -290,24 +294,12 @@ def test_add_view_creates_crawl_with_tag_and_url_filter_overrides(client, admin_
 
 def _csrf_test_add_data(url: str) -> dict[str, str]:
     return {
+        **ADD_FORM_DEFAULTS,
         "url": url,
-        "tag": "",
-        "depth": "0",
-        "max_urls": "0",
-        "crawl_max_size": "0",
         "crawl_timeout": "0",
         "timeout": "",
-        "snapshot_max_size": "0",
         "delete_after": "0",
         "crawl_max_concurrent_snapshots": "1",
-        "url_filters_allowlist": "",
-        "url_filters_denylist": "",
-        "notes": "",
-        "schedule": "",
-        "persona": "Default",
-        "permissions": "public",
-        "start_paused": "",
-        "config": "{}",
     }
 
 
@@ -337,24 +329,14 @@ def test_add_view_sanitizes_crawl_notes_before_safe_update(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/notes-xss",
-            "tag": "",
-            "depth": "0",
             "max_urls": "1",
-            "crawl_max_size": "0",
             "crawl_timeout": "0",
             "timeout": "",
-            "snapshot_max_size": "0",
             "delete_after": "0",
             "crawl_max_concurrent_snapshots": "1",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "notes": malicious_notes,
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -373,20 +355,8 @@ def test_add_view_unchecked_only_new_sets_crawl_override(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/rearchive",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -407,20 +377,10 @@ def test_add_view_selected_persona_wins_over_stale_config_override(client, admin
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/private",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
             "persona": "Private",
-            "permissions": "public",
-            "start_paused": "",
             "config": '{"DEFAULT_PERSONA": "Default"}',
         },
         HTTP_HOST=ADMIN_HOST,
@@ -447,20 +407,9 @@ def test_add_view_applies_plugin_config_overrides(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/plugin-config",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
             "main_plugins": ["wget"],
             "plugin_config__wget__WGET_ENABLED": "false",
             "plugin_config__wget__WGET_TIMEOUT": "77",
@@ -488,9 +437,8 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/public-safe",
-            "tag": "",
-            "depth": "0",
             "max_urls": "10",
             "crawl_max_size": "45mb",
             "crawl_timeout": "120",
@@ -503,8 +451,6 @@ def test_add_view_public_submission_ignores_plugin_and_custom_config(client, adm
             "url_filters_only_new": "1",
             "notes": "public add",
             "schedule": "daily",
-            "persona": "Default",
-            "permissions": "public",
             "start_paused": "on",
             "main_plugins": ["wget"],
             "plugin_config__twocaptcha__TWOCAPTCHA_API_KEY": "posted-token",
@@ -541,21 +487,9 @@ def test_add_view_queues_crawl_for_background_runner(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com",
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -575,21 +509,11 @@ def test_add_view_start_paused_creates_paused_crawl_without_snapshots(client, ad
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": "https://example.com/paused",
-            "tag": "",
             "depth": "1",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
             "start_paused": "on",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -610,6 +534,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user):
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": (
                 "https://sweeting.me,https://google.com\n"
                 "Notes: [ArchiveBox](https://github.com/ArchiveBox/ArchiveBox), https://news.ycombinator.com\n"
@@ -617,20 +542,7 @@ def test_add_view_extracts_urls_from_mixed_text_input(client, admin_user):
                 '{"items":["https://example.com/three"]}\n'
                 "csv,https://example.com/four"
             ),
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )
@@ -656,21 +568,9 @@ def test_add_view_trims_trailing_punctuation_from_markdown_urls(client, admin_us
     response = client.post(
         reverse("add"),
         data={
+            **ADD_FORM_DEFAULTS,
             "url": ("Docs: https://github.com/ArchiveBox/ArchiveBox.\nIssue: https://github.com/abc?abc#234234?."),
-            "tag": "",
-            "depth": "0",
-            "max_urls": "0",
-            "crawl_max_size": "0",
-            "snapshot_max_size": "0",
-            "url_filters_allowlist": "",
-            "url_filters_denylist": "",
             "url_filters_only_new": "1",
-            "notes": "",
-            "schedule": "",
-            "persona": "Default",
-            "permissions": "public",
-            "start_paused": "",
-            "config": "{}",
         },
         HTTP_HOST=ADMIN_HOST,
     )

archivebox/tests/test_ui_add_view_runtime.py

@@ -431,6 +431,10 @@ def test_public_add_view_import_text_formats_preserve_metadata_and_resume_withou
         URL_ALLOWLIST=r"127\.0\.0\.1[:/].*",
     )
     create_admin_and_token(tmp_path)
+    # These tests deliberately stop the server while crawls are queued. Finish
+    # dependency installation first so shutdown cannot leave a half-built venv.
+    install_result = run_archivebox_cmd(["install", *env["PLUGINS"].split(",")], cwd=tmp_path, env=env, timeout=180)
+    assert install_result.returncode == 0, install_result.stderr or install_result.stdout
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)
@@ -536,6 +540,10 @@ def test_public_add_view_rejects_file_path_and_shell_injection_payloads(tmp_path
         URL_ALLOWLIST=r"example\.com|example\.org|iana\.org|www\.iana\.org",
     )
     create_admin_and_token(tmp_path)
+    # These tests deliberately stop the server while crawls are queued. Finish
+    # dependency installation first so shutdown cannot leave a half-built venv.
+    install_result = run_archivebox_cmd(["install", *env["PLUGINS"].split(",")], cwd=tmp_path, env=env, timeout=180)
+    assert install_result.returncode == 0, install_result.stderr or install_result.stdout
 
     try:
         start_archivebox_server(tmp_path, env=env, port=port)

archivebox/tests/test_ui_admin_snapshot.py

@@ -607,7 +607,14 @@ class TestSnapshotProgressStats:
 
         assert result.embed_path_db() is None
 
-    def test_embed_path_db_prefers_valid_output_str_over_first_output_file(self, snapshot, real_hash_projection):
+    @pytest.mark.parametrize(
+        "output_str",
+        [
+            ("wget/example.com/index.html"),
+            (""),
+        ],
+    )
+    def test_embed_path_db_selects_html_output(self, snapshot, real_hash_projection, output_str):
         from archivebox.core.models import ArchiveResult
 
         output_dir = Path(snapshot.output_dir) / "wget" / "example.com" / "assets" / "css"
@@ -619,32 +626,7 @@ class TestSnapshotProgressStats:
         _process, result = real_hash_projection
         ArchiveResult.objects.filter(pk=result.pk).update(
             plugin="wget",
-            output_str="wget/example.com/index.html",
-            output_files={
-                "example.com/assets/css/mobile.css": {"size": (output_dir / "mobile.css").stat().st_size, "mimetype": "text/css"},
-                "example.com/index.html": {
-                    "size": (Path(snapshot.output_dir) / "wget" / "example.com" / "index.html").stat().st_size,
-                    "mimetype": "text/html",
-                },
-            },
-        )
-        result.refresh_from_db()
-
-        assert result.embed_path_db() == "wget/example.com/index.html"
-
-    def test_embed_path_db_scores_output_files_instead_of_using_first_entry(self, snapshot, real_hash_projection):
-        from archivebox.core.models import ArchiveResult
-
-        output_dir = Path(snapshot.output_dir) / "wget" / "example.com" / "assets" / "css"
-        output_dir.mkdir(parents=True, exist_ok=True)
-        (Path(snapshot.output_dir) / "wget" / "example.com" / "index.html").parent.mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(REPO_ROOT / "README.md", Path(snapshot.output_dir) / "wget" / "example.com" / "index.html")
-        shutil.copyfile(REPO_ROOT / "archivebox" / "templates" / "static" / "bootstrap.min.css", output_dir / "mobile.css")
-
-        _process, result = real_hash_projection
-        ArchiveResult.objects.filter(pk=result.pk).update(
-            plugin="wget",
-            output_str="",
+            output_str=output_str,
             output_files={
                 "example.com/assets/css/mobile.css": {"size": (output_dir / "mobile.css").stat().st_size, "mimetype": "text/css"},
                 "example.com/index.html": {

etc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archivebox",
-  "version": "0.9.35rc461",
+  "version": "0.9.35rc467",
   "repository": "github:ArchiveBox/ArchiveBox",
   "license": "MIT",
   "dependencies": {

publicsite/index.html

@@ -54,11 +54,9 @@
       <a href="#use-cases"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"/><circle cx="9" cy="7" r="4"/><path d="M22 21v-2a4 4 0 0 0-3-3.87"/><path d="M16 3.13a4 4 0 0 1 0 7.75"/></svg><span>Use cases</span></a>
       <a href="#configuration"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 21v-7"/><path d="M4 10V3"/><path d="M12 21v-9"/><path d="M12 8V3"/><path d="M20 21v-5"/><path d="M20 12V3"/><path d="M2 14h4"/><path d="M10 8h4"/><path d="M18 16h4"/></svg><span>Configuration</span></a>
       <a href="#layout-security"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10Z"/><path d="m9 12 2 2 4-4"/></svg><span>Layout &amp; security</span></a>
-      <a href="#background"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 8v4l3 3"/><circle cx="12" cy="12" r="9"/></svg><span>Backstory</span></a>
-      <a href="#ecosystem"><svg viewBox="0 0 24 24" aria-hidden="true"><rect x="3" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="14" width="7" height="7" rx="1.5"/><path d="M10 6.5h4"/><path d="M17.5 10v4"/><path d="M10 17.5h4"/></svg><span>Ecosystem</span></a>
+      <details class="project-menu"><summary><svg viewBox="0 0 24 24" aria-hidden="true"><rect x="3" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="3" width="7" height="7" rx="1.5"/><rect x="14" y="14" width="7" height="7" rx="1.5"/><path d="M10 6.5h4"/><path d="M17.5 10v4"/><path d="M10 17.5h4"/></svg><span>Apps</span></summary><div class="project-menu__links"><a href="https://archivebox.io/" aria-current="page">ArchiveBox</a><a href="https://archivebox.github.io/ios-archivebox/">Apple apps</a><a href="https://archivebox.github.io/archivebox-browser-extension/">Browser extension</a><a href="https://archivebox.github.io/abx-dl/">abx-dl</a><a href="https://archivebox.github.io/abx-plugins/">abx-plugins</a></div></details>
       <a href="https://github.com/ArchiveBox/ArchiveBox/wiki"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 19.5A2.5 2.5 0 0 1 6.5 17H20"/><path d="M6.5 2H20v20H6.5A2.5 2.5 0 0 1 4 19.5v-15A2.5 2.5 0 0 1 6.5 2Z"/></svg><span>Docs</span></a>
       <a href="https://github.com/ArchiveBox/ArchiveBox"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M15 22v-4a4.8 4.8 0 0 0-.09-.93"/><path d="M9 22v-4a4.8 4.8 0 0 1 .09-.93"/><path d="M17.8 20.2c1.4-.5 2.2-1.7 2.2-3.7V12.7c0-.7-.2-1.4-.7-2 .1-.3.3-1.7-.1-3.2 0 0-1.1-.4-3.4.9a11.5 11.5 0 0 0-6 0C7.5 7.1 6.4 7.5 6.4 7.5c-.4 1.5-.2 2.9-.1 3.2-.5.6-.7 1.3-.7 2v3.8c0 2 1 3.2 2.4 3.7"/><path d="M9 18c-3 1.3-5 0-6-2"/></svg><span>GitHub</span></a>
-      <a href="https://pypi.org/project/archivebox/"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M16.5 9.4 7.5 4.2"/><path d="M21 16V8a2 2 0 0 0-1-1.73l-7-4a2 2 0 0 0-2 0l-7 4A2 2 0 0 0 3 8v8a2 2 0 0 0 1 1.73l7 4a2 2 0 0 0 2 0l7-4A2 2 0 0 0 21 16Z"/><path d="M3.3 7 12 12l8.7-5"/><path d="M12 22V12"/></svg><span>PyPI</span></a>
     </nav>
   </header>
 

publicsite/styles.css

@@ -125,7 +125,8 @@ code {
   gap: 8px 18px;
 }
 
-.site-header nav a {
+.site-header nav a,
+.site-header nav .project-menu > summary {
   display: inline-flex;
   align-items: center;
   gap: 8px;
@@ -140,7 +141,8 @@ code {
   transition: border-color 160ms ease, color 160ms ease;
 }
 
-.site-header nav a:hover {
+.site-header nav a:hover,
+.site-header nav .project-menu > summary:hover {
   color: var(--text);
   text-decoration: none;
   border-color: hsl(var(--primary) / 0.42);
@@ -806,7 +808,7 @@ h3 {
   }
 
   .site-header {
-    position: static;
+    position: relative;
     width: 100%;
     margin: 0;
     padding: 14px 16px;
@@ -816,7 +818,8 @@ h3 {
     width: 100%;
   }
 
-  .site-header nav a {
+  .site-header nav a,
+  .site-header nav .project-menu > summary {
     font-size: 0.74rem;
     letter-spacing: 0.1em;
   }
@@ -881,3 +884,16 @@ h3 {
     font-size: 0.82rem;
   }
 }
+
+/* Shared project navigation stays usable without JavaScript. */
+.project-menu { position: relative; flex: 0 0 auto; }
+.project-menu > summary { cursor: pointer; list-style: none; }
+.project-menu > summary::-webkit-details-marker { display: none; }
+.project-menu > summary::after { content: "▾"; }
+.project-menu > summary:focus-visible, .project-menu__links a:focus-visible { outline: 3px solid #9b2854; outline-offset: 3px; }
+.project-menu__links { position: absolute; right: 0; top: 100%; z-index: 30; width: 220px; max-width: calc(100vw - 32px); padding: 8px; border: 1px solid #e9e2e3; border-radius: 12px; background: #fff; box-shadow: 0 12px 32px #28252a20; }
+header nav .project-menu__links a { display: flex; align-items: center; min-height: 44px; padding: 8px 12px; color: #28252a; font-size: 13px; letter-spacing: normal; text-transform: none; text-decoration: none; white-space: nowrap; }
+header nav .project-menu__links a:hover, header nav .project-menu__links a[aria-current="page"] { background: #f8edf2; color: #9b2854; border-radius: 6px; }
+@media (max-width: 700px) { header.site-header { flex-wrap: wrap; } header.site-header nav { flex-wrap: wrap; gap: 8px 16px; } }
+
+@media (max-width: 1100px) { .project-menu__links { position: fixed; left: 16px; right: 16px; top: auto; width: auto; } }

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "archivebox"
-version = "0.9.35rc461"
+version = "0.9.35rc467"
 requires-python = ">=3.13"
 description = "Self-hosted internet archiving solution."
 authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -79,8 +79,8 @@ dependencies = [
     ### Binary/Package Management
     "abxbus==2.5.56",                 # direct imports only; version constrained by abx-dl -> abx-plugins
     "abxpkg==1.12.117",                # direct imports only; version constrained by abx-dl -> abx-plugins
-    "abx-plugins[opencode]==1.12.244", # optional AI runtime clients are owned by the plugin
-    "abx-dl==1.12.282",       # shared ArchiveBox downloader package
+    "abx-plugins[opencode]==1.12.249", # optional AI runtime clients are owned by the plugin
+    "abx-dl==1.12.285",       # shared ArchiveBox downloader package
     ### UUID7 backport for Python <3.14
     "uuid7>=0.1.0; python_version < '3.14'",  # provides the uuid_extensions module on Python 3.13
     "browser-cookie3>=0.20.1",
@@ -342,7 +342,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
 
 
 [tool.bumpver]
-current_version = "v0.9.35rc461"
+current_version = "v0.9.35rc467"
 version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "bump version {old_version} -> {new_version}"
 tag_message = "{new_version}"

uv.lock

@@ -13,18 +13,18 @@ supported-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-12T23:41:52.996600165Z"
+exclude-newer = "2026-09-13T04:51:40.198736069Z"
 exclude-newer-span = "P5D"
 
 [options.exclude-newer-package]
-abxbus = { timestamp = "2026-09-17T23:41:51.996613044Z", span = "PT1S" }
+abxbus = { timestamp = "2026-09-18T04:51:39.198748378Z", span = "PT1S" }
 abx-plugins = "2100-01-01T00:00:00Z"
 abx-dl = "2100-01-01T00:00:00Z"
 abxpkg = "2100-01-01T00:00:00Z"
 
 [[package]]
 name = "abx-dl"
-version = "1.12.282"
+version = "1.12.285"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
     { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e8/61/6cb403767815780042875c68137a924e1a20862a50c9e7eefe7c09671817/abx_dl-1.12.282.tar.gz", hash = "sha256:3c49774432a3777c4441d5306ad174019653f8a668f46b1ee57258c5f65d6361", size = 88227 }
+sdist = { url = "https://files.pythonhosted.org/packages/78/7f/c02c8e95a17427e6258d789ee943d029195188fede7fc420df263a1f6b66/abx_dl-1.12.285.tar.gz", hash = "sha256:84f9dd4ae3b58fb2e7923c67905583f39d163e0e95991ea9d2da9f71cd31b783", size = 88234 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a8/b6/a19d8a07aa4e375c880101d99d8a63749b5cc6cb1542ea4761a9d233f089/abx_dl-1.12.282-py3-none-any.whl", hash = "sha256:7a922ee8b35b711c26c489549845b49897c712664f47bc08b8a85f2bd8577a8d", size = 93263 },
+    { url = "https://files.pythonhosted.org/packages/a6/e0/70774861b35ddd4b52c99253d83c8d2a7be039e8856ee1cfcaa4b662b05a/abx_dl-1.12.285-py3-none-any.whl", hash = "sha256:e729ebe51dc4f23c546d0ab7743100de821c03d56ecb4fba7ff3d6c50e7bf4f6", size = 93263 },
 ]
 
 [[package]]
 name = "abx-plugins"
-version = "1.12.244"
+version = "1.12.249"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
     { name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/03/bf/dfa79260a5d019452a912371c2fedee9cf7cc85aeadf0f14c6110b1c2dca/abx_plugins-1.12.244.tar.gz", hash = "sha256:c9f0170fcd0e1a83aada3f425c0999dcb9355ae75dfd6410a1cab32f15c1789b", size = 293567, upload-time = "2026-09-17T23:27:22.652Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/56/ec/92145ef9893b5e2e83905997d1e824fc521f304333d895f0b9d81f349d5b/abx_plugins-1.12.249.tar.gz", hash = "sha256:25438e5f4cc3942b9cc0dc6d857c0243309d444a7e97aaf62818e040cf124c02", size = 309461, upload-time = "2026-09-18T04:30:46.963Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/1e/1b/5056288814dc2c07dde0f220ca9d1871ae8c508d5602a5aa6bb7cc1bcf88/abx_plugins-1.12.244-py3-none-any.whl", hash = "sha256:92f3cd8fbab5ee248e6460deda715225a165386693acd5c783571a56bc4d7acd", size = 457740, upload-time = "2026-09-17T23:27:20.918Z" },
+    { url = "https://files.pythonhosted.org/packages/24/eb/8ea746a247bb195bba545e678133774830a4e14ec36f97fb353f0345d44e/abx_plugins-1.12.249-py3-none-any.whl", hash = "sha256:0445c8cba4263845a04d7b30c8c2e58da33feb52c4dba477c19e5f219c8e3f39", size = 473839, upload-time = "2026-09-18T04:30:48.418Z" },
 ]
 
 [package.optional-dependencies]
@@ -127,7 +127,7 @@ wheels = [
 
 [[package]]
 name = "archivebox"
-version = "0.9.35rc461"
+version = "0.9.35rc467"
 source = { editable = "." }
 dependencies = [
     { name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -226,8 +226,8 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "abx-dl", specifier = "==1.12.282" },
-    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.12.244" },
+    { name = "abx-dl", specifier = "==1.12.285" },
+    { name = "abx-plugins", extras = ["opencode"], specifier = "==1.12.249" },
     { name = "abxbus", specifier = "==2.5.56" },
     { name = "abxpkg", specifier = "==1.12.117" },
     { name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },