Compare commits
175
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3fcda80297 | ||
|
|
e94a489daa | ||
|
|
df605452ba | ||
|
|
b472401d2f | ||
|
|
e4a1e400e8 | ||
|
|
2ce7074118 | ||
|
|
03bd48e399 | ||
|
|
edd1265b42 | ||
|
|
6bafaa99d7 | ||
|
|
53e3e9787b | ||
|
|
8e82de7f4a | ||
|
|
41dfc384aa | ||
|
|
66d88542f6 | ||
|
|
268b520394 | ||
|
|
3a3557ac72 | ||
|
|
b8ba9a64cd | ||
|
|
58a7955835 | ||
|
|
564af2fa3e | ||
|
|
ccaa630821 | ||
|
|
a3c53aca45 | ||
|
|
e0e3a7d09b | ||
|
|
c97cea180e | ||
|
|
5efc69d04b | ||
|
|
912713a732 | ||
|
|
f4c84ff6ac | ||
|
|
3ced9981f7 | ||
|
|
49204a84e4 | ||
|
|
da1665acb3 | ||
|
|
5f20275598 | ||
|
|
8bbd83ea3f | ||
|
|
4c847dff79 | ||
|
|
2e2aaf50ed | ||
|
|
9f8e58a0a0 | ||
|
|
6b46a328f0 | ||
|
|
b1cdac35d6 | ||
|
|
33c4801abf | ||
|
|
f464d764de | ||
|
|
d342c38998 | ||
|
|
f1f9a14601 | ||
|
|
de368a9c6a | ||
|
|
0a4249b63a | ||
|
|
a17c7401df | ||
|
|
84222105ec | ||
|
|
bf505c0dc1 | ||
|
|
182b82cd40 | ||
|
|
e586110299 | ||
|
|
d17ebca602 | ||
|
|
2e791f228a | ||
|
|
22553aec7b | ||
|
|
4aa46feec6 | ||
|
|
a836fc2574 | ||
|
|
b6207952d2 | ||
|
|
62c942aac8 | ||
|
|
0887494cf0 | ||
|
|
f7b5141a9c | ||
|
|
a8cf5140bc | ||
|
|
f61a59f6cf | ||
|
|
abf6a63add | ||
|
|
798676c2a2 | ||
|
|
603327065d | ||
|
|
00df378b23 | ||
|
|
479f2e90ad | ||
|
|
c38dc8845d | ||
|
|
af66f9cb04 | ||
|
|
f83619cda0 | ||
|
|
b6e0f7cc3b | ||
|
|
9a8457e9da | ||
|
|
97e210e7ad | ||
|
|
339950051b | ||
|
|
e78348a89b | ||
|
|
0f96eee462 | ||
|
|
8703e42474 | ||
|
|
7efb88d5f5 | ||
|
|
62b6f11f66 | ||
|
|
47737a5a3e | ||
|
|
4898364e38 | ||
|
|
1a6d2055b5 | ||
|
|
ab92d4098b | ||
|
|
166a42a923 | ||
|
|
24438d981f | ||
|
|
424d39c17e | ||
|
|
814edf8f0d | ||
|
|
bf5730fe49 | ||
|
|
6094135eaf | ||
|
|
bf09b41bbb | ||
|
|
fcb34a20ee | ||
|
|
c59b8dc3ff | ||
|
|
da74c8a54b | ||
|
|
8cfbcd5177 | ||
|
|
1c7dcef2cf | ||
|
|
4bfe825909 | ||
|
|
815c274bd9 | ||
|
|
6c18b1b4dc | ||
|
|
ce644b89c4 | ||
|
|
a369661d96 | ||
|
|
a8fca2f089 | ||
|
|
d85e6fb02a | ||
|
|
be20060acd | ||
|
|
7e0634d470 | ||
|
|
28cda38e61 | ||
|
|
0c30d2cf6a | ||
|
|
868080c7d3 | ||
|
|
fad2bc2491 | ||
|
|
61baddce3c | ||
|
|
bb2483e94d | ||
|
|
69353748f3 | ||
|
|
1cc6c74458 | ||
|
|
68abcc6bd3 | ||
|
|
6b3fa1c35c | ||
|
|
02560e66f1 | ||
|
|
b8c5a9f692 | ||
|
|
bdff9f8acd | ||
|
|
85e4b004f9 | ||
|
|
9961d13264 | ||
|
|
8017554279 | ||
|
|
d5b32a2a88 | ||
|
|
7acac48adc | ||
|
|
c2122711e8 | ||
|
|
026987acc8 | ||
|
|
797af67c8b | ||
|
|
fbb10ca42b | ||
|
|
73fd04983d | ||
|
|
d2584af923 | ||
|
|
1104598c78 | ||
|
|
8d543bd8e3 | ||
|
|
4162ff805b | ||
|
|
66164455da | ||
|
|
f617459a28 | ||
|
|
8bd911a8b9 | ||
|
|
6369ec46f0 | ||
|
|
40df1a60a6 | ||
|
|
9c0bf77b93 | ||
|
|
d6f1933fc1 | ||
|
|
2dcc94629d | ||
|
|
da5580028a | ||
|
|
3697324490 | ||
|
|
4e36f10d82 | ||
|
|
d39853202b | ||
|
|
bd8d8b6bb9 | ||
|
|
fb0c8d8fbc | ||
|
|
9ee32a8315 | ||
|
|
8a2063f61c | ||
|
|
84538a603e | ||
|
|
432fb1b152 | ||
|
|
01bf416811 | ||
|
|
9612d3b185 | ||
|
|
b5a53a5778 | ||
|
|
4adc610218 | ||
|
|
27bf63315c | ||
|
|
b34c8f6899 | ||
|
|
752a2cfff1 | ||
|
|
0ab7bc808e | ||
|
|
7b5520e84d | ||
|
|
ee0bdc18bf | ||
|
|
ee54fbb48e | ||
|
|
7d852f4dac | ||
|
|
a9e3d04913 | ||
|
|
da37976838 | ||
|
|
8405e6575f | ||
|
|
bb88eaa874 | ||
|
|
bece3e0de4 | ||
|
|
2d3a19ff5e | ||
|
|
875480d29c | ||
|
|
e4efe57e9b | ||
|
|
71c161b410 | ||
|
|
39539b1093 | ||
|
|
5d6c73b624 | ||
|
|
181ad0e94d | ||
|
|
54e4de164d | ||
|
|
1640e6e54e | ||
|
|
a293c0df82 | ||
|
|
1ff6326eee | ||
|
|
9687561c47 | ||
|
|
2a14c50acf | ||
|
|
171a782cec |
No files matched your search
@@ -65,10 +65,15 @@ Build: `uv run --no-project python .github/pages/site.py render _site --source p
|
||||
Check the built site with real Chromium (also run by Pages CI):
|
||||
|
||||
```sh
|
||||
uv run --no-config --no-project --with playwright==1.63.0 playwright install chromium
|
||||
export ABXPKG_LIB_DIR=/tmp/archivebox-site-browser
|
||||
CHROME_SCHEMA="$(uv run python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
|
||||
uv run abxpkg env --install --deps-from="$CHROME_SCHEMA:required_binaries"
|
||||
export CHROME_BINARY="$ABXPKG_LIB_DIR/env/bin/chromium"
|
||||
uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py SITE_OUTPUT --evidence /tmp/site-evidence
|
||||
```
|
||||
|
||||
The browser comes from the normal abxpkg provider/cache path; Playwright drives
|
||||
that resolved executable instead of installing a second Chromium build.
|
||||
Use this workflow's `SITE_OUTPUT` directory. The check covers desktop/mobile
|
||||
layout, keyboard dismissal, no-JavaScript links, local resources and visible
|
||||
images, and uploads its screenshots as `site-verification` for 14 days.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from functools import partial
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
@@ -75,7 +76,7 @@ def verify(output, evidence):
|
||||
origin = f"http://127.0.0.1:{server.server_port}"
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(ignore_default_args=["--hide-scrollbars"])
|
||||
browser = playwright.chromium.launch(executable_path=os.environ.get("CHROME_BINARY"), ignore_default_args=["--hide-scrollbars"])
|
||||
for name in config["pages"]:
|
||||
route = name.removesuffix("index.html")
|
||||
page = browser.new_page(reduced_motion="reduce")
|
||||
|
||||
@@ -13,8 +13,9 @@ def main() -> None:
|
||||
if not archivebox_tests:
|
||||
raise SystemExit("No ArchiveBox tests discovered")
|
||||
|
||||
# These are observed whole-job seconds, including setup. Unknown/long files,
|
||||
# optional dependencies and hosted-only requirements keep independent jobs.
|
||||
# These are observed per-file pytest seconds, including fixture/startup
|
||||
# time. Shared job setup is paid once per batch. Unknown/long files, optional
|
||||
# dependencies and hosted-only requirements keep independent jobs.
|
||||
durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
|
||||
hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
|
||||
requested = os.environ.get("TEST_FILE", "")
|
||||
|
||||
+170
-165
@@ -1,171 +1,176 @@
|
||||
{
|
||||
"source_runs": [
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/36302511923",
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552",
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/36454992009",
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/37186118498"
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/37190995942",
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/37237793977"
|
||||
],
|
||||
"measurement": "Whole job seconds, including setup and excluding queue time. Hosted unbatched files use the latest complete successful run; other files retain their individual baseline measurement. test_issue_1896 uses its successful ugNAS job measurement from run 37186118498. Unknown files stay independent.",
|
||||
"measurement": "Unbatched files use successful per-file pytest step wall seconds from run 37237793977, including process and fixture startup, on their actual hosted or ugNAS runner. Already-batched files retain prior JUnit measurements plus 9 seconds startup. Queue and shared job setup are excluded; unknown files remain independent.",
|
||||
"seconds": {
|
||||
"archivebox/tests/test_api_v1_auth_browser_session.py": 29,
|
||||
"archivebox/tests/test_api_v1_auth_check_api_token.py": 27,
|
||||
"archivebox/tests/test_api_v1_auth_get_api_token.py": 24,
|
||||
"archivebox/tests/test_api_v1_cli_add.py": 96,
|
||||
"archivebox/tests/test_api_v1_cli_remove.py": 50,
|
||||
"archivebox/tests/test_api_v1_cli_schedule.py": 41,
|
||||
"archivebox/tests/test_api_v1_cli_search.py": 26,
|
||||
"archivebox/tests/test_api_v1_cli_update.py": 74,
|
||||
"archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 64,
|
||||
"archivebox/tests/test_api_v1_core_any_id.py": 21,
|
||||
"archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 55,
|
||||
"archivebox/tests/test_api_v1_core_archiveresults.py": 32,
|
||||
"archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 199,
|
||||
"archivebox/tests/test_api_v1_core_snapshots_rss.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_snapshots.py": 28,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 23,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 28,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_tags_autocomplete.py": 26,
|
||||
"archivebox/tests/test_api_v1_core_tags_create.py": 21,
|
||||
"archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 25,
|
||||
"archivebox/tests/test_api_v1_core_tags_search.py": 26,
|
||||
"archivebox/tests/test_api_v1_core_tags.py": 25,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 22,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 25,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 24,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 86,
|
||||
"archivebox/tests/test_api_v1_crawls_crawls.py": 21,
|
||||
"archivebox/tests/test_api_v1_machine_binaries.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_binary_binary_id.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 26,
|
||||
"archivebox/tests/test_api_v1_machine_machine_current.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_machine_machine_id.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_machines.py": 27,
|
||||
"archivebox/tests/test_api_v1_personas_personas.py": 23,
|
||||
"archivebox/tests/test_api_v1_personas_sync.py": 29,
|
||||
"archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 36,
|
||||
"archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 33,
|
||||
"archivebox/tests/test_archive_result_service.py": 137,
|
||||
"archivebox/tests/test_auth_ldap.py": 44,
|
||||
"archivebox/tests/test_binary_service.py": 83,
|
||||
"archivebox/tests/test_cli_add_1.py": 168,
|
||||
"archivebox/tests/test_cli_add_2.py": 203,
|
||||
"archivebox/tests/test_cli_add_3.py": 215,
|
||||
"archivebox/tests/test_cli_add_4.py": 201,
|
||||
"archivebox/tests/test_cli_archiveresult.py": 199,
|
||||
"archivebox/tests/test_cli_binary.py": 20,
|
||||
"archivebox/tests/test_cli_config.py": 148,
|
||||
"archivebox/tests/test_cli_crawl.py": 177,
|
||||
"archivebox/tests/test_cli_extract_input.py": 155,
|
||||
"archivebox/tests/test_cli_extract.py": 106,
|
||||
"archivebox/tests/test_cli_help.py": 30,
|
||||
"archivebox/tests/test_cli_init.py": 135,
|
||||
"archivebox/tests/test_cli_install.py": 58,
|
||||
"archivebox/tests/test_cli_list.py": 260,
|
||||
"archivebox/tests/test_cli_machine.py": 50,
|
||||
"archivebox/tests/test_cli_manage.py": 48,
|
||||
"archivebox/tests/test_cli_mcp.py": 51,
|
||||
"archivebox/tests/test_cli_oneshot.py": 23,
|
||||
"archivebox/tests/test_cli_persona.py": 82,
|
||||
"archivebox/tests/test_cli_piping.py": 97,
|
||||
"archivebox/tests/test_cli_pluginmap.py": 21,
|
||||
"archivebox/tests/test_cli_process.py": 20,
|
||||
"archivebox/tests/test_cli_remove.py": 162,
|
||||
"archivebox/tests/test_cli_run_1.py": 195,
|
||||
"archivebox/tests/test_cli_run_2.py": 255,
|
||||
"archivebox/tests/test_cli_run_3.py": 167,
|
||||
"archivebox/tests/test_cli_run_4.py": 178,
|
||||
"archivebox/tests/test_cli_schedule.py": 87,
|
||||
"archivebox/tests/test_cli_search.py": 35,
|
||||
"archivebox/tests/test_cli_server.py": 259,
|
||||
"archivebox/tests/test_cli_shell.py": 32,
|
||||
"archivebox/tests/test_cli_snapshot.py": 181,
|
||||
"archivebox/tests/test_cli_status.py": 102,
|
||||
"archivebox/tests/test_cli_tag.py": 21,
|
||||
"archivebox/tests/test_cli_update_reindex_snapshots.py": 163,
|
||||
"archivebox/tests/test_cli_update.py": 142,
|
||||
"archivebox/tests/test_cli_version.py": 143,
|
||||
"archivebox/tests/test_config_DELETE_AFTER.py": 169,
|
||||
"archivebox/tests/test_config_MAX_limits.py": 34,
|
||||
"archivebox/tests/test_config_ONLY_NEW.py": 24,
|
||||
"archivebox/tests/test_config_SAVE_TITLE.py": 99,
|
||||
"archivebox/tests/test_config_URL_filters.py": 26,
|
||||
"archivebox/tests/test_core_config.py": 20,
|
||||
"archivebox/tests/test_core_forms.py": 28,
|
||||
"archivebox/tests/test_crawl_pause.py": 20,
|
||||
"archivebox/tests/test_crawl_runner.py": 45,
|
||||
"archivebox/tests/test_crawl_service.py": 34,
|
||||
"archivebox/tests/test_frozen_crawl_config.py": 47,
|
||||
"archivebox/tests/test_hook_result_lifecycle.py": 78,
|
||||
"archivebox/tests/test_hooks.py": 90,
|
||||
"archivebox/tests/test_issue_1896.py": 485,
|
||||
"archivebox/tests/test_machine_models.py": 52,
|
||||
"archivebox/tests/test_machine_service.py": 89,
|
||||
"archivebox/tests/test_migrations_04_to_09.py": 36,
|
||||
"archivebox/tests/test_migrations_07_to_09.py": 183,
|
||||
"archivebox/tests/test_migrations_08_to_09.py": 244,
|
||||
"archivebox/tests/test_migrations_fresh.py": 99,
|
||||
"archivebox/tests/test_misc_checks.py": 22,
|
||||
"archivebox/tests/test_misc_monkey_patches.py": 20,
|
||||
"archivebox/tests/test_opencode_agent_1.py": 158,
|
||||
"archivebox/tests/test_opencode_agent_2.py": 145,
|
||||
"archivebox/tests/test_opencode_agent_3.py": 154,
|
||||
"archivebox/tests/test_opencode_browser_1.py": 262,
|
||||
"archivebox/tests/test_opencode_browser_2.py": 146,
|
||||
"archivebox/tests/test_opencode_browser_3.py": 148,
|
||||
"archivebox/tests/test_orm_helpers.py": 22,
|
||||
"archivebox/tests/test_permissions.py": 21,
|
||||
"archivebox/tests/test_persona_runtime.py": 56,
|
||||
"archivebox/tests/test_postgres_backend.py": 85,
|
||||
"archivebox/tests/test_process_runtime_paths.py": 28,
|
||||
"archivebox/tests/test_process_service.py": 49,
|
||||
"archivebox/tests/test_proof_previews.py": 54,
|
||||
"archivebox/tests/test_recursive_crawl_1.py": 339,
|
||||
"archivebox/tests/test_recursive_crawl_2.py": 171,
|
||||
"archivebox/tests/test_recursive_crawl_3.py": 176,
|
||||
"archivebox/tests/test_release_notes.py": 14,
|
||||
"archivebox/tests/test_release_source_fingerprint.py": 22,
|
||||
"archivebox/tests/test_release_workflow.py": 21,
|
||||
"archivebox/tests/test_resource_admission.py": 21,
|
||||
"archivebox/tests/test_runner_watch.py": 177,
|
||||
"archivebox/tests/test_search.py": 153,
|
||||
"archivebox/tests/test_serve_static.py": 31,
|
||||
"archivebox/tests/test_server_automatic_url.py": 36,
|
||||
"archivebox/tests/test_server_security_browser.py": 278,
|
||||
"archivebox/tests/test_settings_signal_webhooks.py": 21,
|
||||
"archivebox/tests/test_setup_script.py": 21,
|
||||
"archivebox/tests/test_shutdown_util.py": 22,
|
||||
"archivebox/tests/test_snapshot_filesystem_migration.py": 29,
|
||||
"archivebox/tests/test_snapshot_output_stacks.py": 62,
|
||||
"archivebox/tests/test_snapshot_ref_lookup.py": 23,
|
||||
"archivebox/tests/test_snapshot_service.py": 41,
|
||||
"archivebox/tests/test_snapshot_url_length.py": 26,
|
||||
"archivebox/tests/test_tag_service.py": 24,
|
||||
"archivebox/tests/test_takeover_util_1.py": 143,
|
||||
"archivebox/tests/test_takeover_util_2.py": 201,
|
||||
"archivebox/tests/test_takeover_util_3.py": 138,
|
||||
"archivebox/tests/test_takeover_util_4.py": 156,
|
||||
"archivebox/tests/test_test_harness.py": 22,
|
||||
"archivebox/tests/test_ui_add_view_runtime.py": 221,
|
||||
"archivebox/tests/test_ui_add_view.py": 46,
|
||||
"archivebox/tests/test_ui_admin_apitoken.py": 23,
|
||||
"archivebox/tests/test_ui_admin_archiveresult.py": 30,
|
||||
"archivebox/tests/test_ui_admin_config_widget.py": 25,
|
||||
"archivebox/tests/test_ui_admin_crawl.py": 44,
|
||||
"archivebox/tests/test_ui_admin_links.py": 35,
|
||||
"archivebox/tests/test_ui_admin_machine.py": 29,
|
||||
"archivebox/tests/test_ui_admin_persona.py": 28,
|
||||
"archivebox/tests/test_ui_admin_snapshot.py": 273,
|
||||
"archivebox/tests/test_ui_admin_tag.py": 26,
|
||||
"archivebox/tests/test_ui_admin_user.py": 28,
|
||||
"archivebox/tests/test_ui_config_views.py": 43,
|
||||
"archivebox/tests/test_ui_live_progress.py": 297,
|
||||
"archivebox/tests/test_ui_public_snapshot.py": 179,
|
||||
"archivebox/tests/test_ui_savepagenow.py": 48,
|
||||
"archivebox/tests/test_urls.py": 220,
|
||||
"archivebox/tests/test_util.py": 20,
|
||||
"archivebox/tests/test_version_metadata.py": 24
|
||||
"archivebox/tests/test_api_discovery_performance.py": 28,
|
||||
"archivebox/tests/test_api_v1_auth_browser_session.py": 17,
|
||||
"archivebox/tests/test_api_v1_auth_check_api_token.py": 15,
|
||||
"archivebox/tests/test_api_v1_auth_get_api_token.py": 16,
|
||||
"archivebox/tests/test_api_v1_cli_add.py": 52,
|
||||
"archivebox/tests/test_api_v1_cli_remove.py": 33,
|
||||
"archivebox/tests/test_api_v1_cli_schedule.py": 29,
|
||||
"archivebox/tests/test_api_v1_cli_search.py": 15,
|
||||
"archivebox/tests/test_api_v1_cli_update.py": 60,
|
||||
"archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 59,
|
||||
"archivebox/tests/test_api_v1_core_any_id.py": 13,
|
||||
"archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 24,
|
||||
"archivebox/tests/test_api_v1_core_archiveresults.py": 19,
|
||||
"archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 106,
|
||||
"archivebox/tests/test_api_v1_core_snapshots.py": 17,
|
||||
"archivebox/tests/test_api_v1_core_snapshots_rss.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tags.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 16,
|
||||
"archivebox/tests/test_api_v1_core_tags_autocomplete.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_create.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 15,
|
||||
"archivebox/tests/test_api_v1_core_tags_search.py": 17,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 69,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 15,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 15,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 16,
|
||||
"archivebox/tests/test_api_v1_crawls_crawls.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_binaries.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_binary_binary_id.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_machine_current.py": 15,
|
||||
"archivebox/tests/test_api_v1_machine_machine_machine_id.py": 16,
|
||||
"archivebox/tests/test_api_v1_machine_machines.py": 15,
|
||||
"archivebox/tests/test_api_v1_personas_personas.py": 15,
|
||||
"archivebox/tests/test_api_v1_personas_sync.py": 16,
|
||||
"archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 27,
|
||||
"archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 19,
|
||||
"archivebox/tests/test_archive_result_service.py": 49,
|
||||
"archivebox/tests/test_auth_ldap.py": 37,
|
||||
"archivebox/tests/test_binary_service.py": 46,
|
||||
"archivebox/tests/test_cli_add_1.py": 75,
|
||||
"archivebox/tests/test_cli_add_2.py": 98,
|
||||
"archivebox/tests/test_cli_add_3.py": 93,
|
||||
"archivebox/tests/test_cli_add_4.py": 94,
|
||||
"archivebox/tests/test_cli_archiveresult.py": 48,
|
||||
"archivebox/tests/test_cli_background_screenshot.py": 49,
|
||||
"archivebox/tests/test_cli_binary.py": 12,
|
||||
"archivebox/tests/test_cli_config.py": 88,
|
||||
"archivebox/tests/test_cli_crawl.py": 81,
|
||||
"archivebox/tests/test_cli_extract.py": 97,
|
||||
"archivebox/tests/test_cli_extract_input.py": 84,
|
||||
"archivebox/tests/test_cli_help.py": 16,
|
||||
"archivebox/tests/test_cli_init.py": 111,
|
||||
"archivebox/tests/test_cli_install.py": 38,
|
||||
"archivebox/tests/test_cli_list.py": 116,
|
||||
"archivebox/tests/test_cli_machine.py": 19,
|
||||
"archivebox/tests/test_cli_manage.py": 32,
|
||||
"archivebox/tests/test_cli_mcp.py": 30,
|
||||
"archivebox/tests/test_cli_oneshot.py": 15,
|
||||
"archivebox/tests/test_cli_persona.py": 56,
|
||||
"archivebox/tests/test_cli_piping.py": 118,
|
||||
"archivebox/tests/test_cli_pluginmap.py": 12,
|
||||
"archivebox/tests/test_cli_process.py": 12,
|
||||
"archivebox/tests/test_cli_remove.py": 46,
|
||||
"archivebox/tests/test_cli_run_1.py": 93,
|
||||
"archivebox/tests/test_cli_run_2.py": 111,
|
||||
"archivebox/tests/test_cli_run_3.py": 51,
|
||||
"archivebox/tests/test_cli_run_4.py": 78,
|
||||
"archivebox/tests/test_cli_schedule.py": 59,
|
||||
"archivebox/tests/test_cli_search.py": 25,
|
||||
"archivebox/tests/test_cli_server.py": 126,
|
||||
"archivebox/tests/test_cli_shell.py": 17,
|
||||
"archivebox/tests/test_cli_snapshot.py": 55,
|
||||
"archivebox/tests/test_cli_status.py": 46,
|
||||
"archivebox/tests/test_cli_tag.py": 12,
|
||||
"archivebox/tests/test_cli_update.py": 62,
|
||||
"archivebox/tests/test_cli_update_reindex_snapshots.py": 84,
|
||||
"archivebox/tests/test_cli_version.py": 81,
|
||||
"archivebox/tests/test_config_DELETE_AFTER.py": 145,
|
||||
"archivebox/tests/test_config_MAX_limits.py": 25,
|
||||
"archivebox/tests/test_config_ONLY_NEW.py": 15,
|
||||
"archivebox/tests/test_config_SAVE_TITLE.py": 59,
|
||||
"archivebox/tests/test_config_URL_filters.py": 15,
|
||||
"archivebox/tests/test_core_config.py": 11,
|
||||
"archivebox/tests/test_core_forms.py": 14,
|
||||
"archivebox/tests/test_crawl_history.py": 60,
|
||||
"archivebox/tests/test_crawl_pause.py": 11,
|
||||
"archivebox/tests/test_crawl_runner.py": 22,
|
||||
"archivebox/tests/test_crawl_runner_2.py": 27,
|
||||
"archivebox/tests/test_crawl_service.py": 19,
|
||||
"archivebox/tests/test_frozen_crawl_config.py": 26,
|
||||
"archivebox/tests/test_hook_result_lifecycle.py": 48,
|
||||
"archivebox/tests/test_hooks.py": 54,
|
||||
"archivebox/tests/test_issue_1896.py": 115,
|
||||
"archivebox/tests/test_machine_models.py": 26,
|
||||
"archivebox/tests/test_machine_service.py": 77,
|
||||
"archivebox/tests/test_migrations_04_to_09.py": 25,
|
||||
"archivebox/tests/test_migrations_07_to_09.py": 86,
|
||||
"archivebox/tests/test_migrations_08_to_09.py": 85,
|
||||
"archivebox/tests/test_migrations_fresh.py": 55,
|
||||
"archivebox/tests/test_misc_checks.py": 12,
|
||||
"archivebox/tests/test_misc_monkey_patches.py": 12,
|
||||
"archivebox/tests/test_opencode_agent_1.py": 73,
|
||||
"archivebox/tests/test_opencode_agent_2.py": 141,
|
||||
"archivebox/tests/test_opencode_agent_3.py": 61,
|
||||
"archivebox/tests/test_opencode_browser_1.py": 139,
|
||||
"archivebox/tests/test_opencode_browser_2.py": 164,
|
||||
"archivebox/tests/test_opencode_browser_3.py": 125,
|
||||
"archivebox/tests/test_orm_helpers.py": 14,
|
||||
"archivebox/tests/test_permissions.py": 12,
|
||||
"archivebox/tests/test_persona_casing.py": 45,
|
||||
"archivebox/tests/test_persona_runtime.py": 30,
|
||||
"archivebox/tests/test_postgres_backend.py": 79,
|
||||
"archivebox/tests/test_process_runtime_paths.py": 20,
|
||||
"archivebox/tests/test_process_service.py": 40,
|
||||
"archivebox/tests/test_proof_previews.py": 24,
|
||||
"archivebox/tests/test_recursive_crawl_1.py": 361,
|
||||
"archivebox/tests/test_recursive_crawl_2.py": 55,
|
||||
"archivebox/tests/test_recursive_crawl_3.py": 67,
|
||||
"archivebox/tests/test_release_notes.py": 12,
|
||||
"archivebox/tests/test_release_source_fingerprint.py": 13,
|
||||
"archivebox/tests/test_release_workflow.py": 15,
|
||||
"archivebox/tests/test_resource_admission.py": 13,
|
||||
"archivebox/tests/test_runner_deletion.py": 91,
|
||||
"archivebox/tests/test_runner_watch.py": 75,
|
||||
"archivebox/tests/test_search.py": 142,
|
||||
"archivebox/tests/test_serve_static.py": 19,
|
||||
"archivebox/tests/test_server_automatic_url.py": 22,
|
||||
"archivebox/tests/test_server_security_browser.py": 121,
|
||||
"archivebox/tests/test_settings_signal_webhooks.py": 13,
|
||||
"archivebox/tests/test_setup_script.py": 12,
|
||||
"archivebox/tests/test_shutdown_util.py": 13,
|
||||
"archivebox/tests/test_snapshot_filesystem_migration.py": 16,
|
||||
"archivebox/tests/test_snapshot_output_stacks.py": 63,
|
||||
"archivebox/tests/test_snapshot_ref_lookup.py": 15,
|
||||
"archivebox/tests/test_snapshot_service.py": 48,
|
||||
"archivebox/tests/test_snapshot_url_length.py": 16,
|
||||
"archivebox/tests/test_tag_service.py": 15,
|
||||
"archivebox/tests/test_takeover_util_1.py": 60,
|
||||
"archivebox/tests/test_takeover_util_2.py": 62,
|
||||
"archivebox/tests/test_takeover_util_3.py": 76,
|
||||
"archivebox/tests/test_takeover_util_4.py": 84,
|
||||
"archivebox/tests/test_test_harness.py": 12,
|
||||
"archivebox/tests/test_ui_add_view.py": 26,
|
||||
"archivebox/tests/test_ui_add_view_runtime.py": 120,
|
||||
"archivebox/tests/test_ui_admin_apitoken.py": 16,
|
||||
"archivebox/tests/test_ui_admin_archiveresult.py": 18,
|
||||
"archivebox/tests/test_ui_admin_config_widget.py": 13,
|
||||
"archivebox/tests/test_ui_admin_crawl.py": 25,
|
||||
"archivebox/tests/test_ui_admin_links.py": 23,
|
||||
"archivebox/tests/test_ui_admin_machine.py": 18,
|
||||
"archivebox/tests/test_ui_admin_persona.py": 19,
|
||||
"archivebox/tests/test_ui_admin_snapshot.py": 138,
|
||||
"archivebox/tests/test_ui_admin_tag.py": 16,
|
||||
"archivebox/tests/test_ui_admin_user.py": 16,
|
||||
"archivebox/tests/test_ui_config_views.py": 20,
|
||||
"archivebox/tests/test_ui_live_progress.py": 121,
|
||||
"archivebox/tests/test_ui_public_snapshot.py": 72,
|
||||
"archivebox/tests/test_ui_savepagenow.py": 31,
|
||||
"archivebox/tests/test_urls.py": 105,
|
||||
"archivebox/tests/test_util.py": 13,
|
||||
"archivebox/tests/test_version_metadata.py": 18,
|
||||
"archivebox/tests/test_zip_preview.py": 29
|
||||
}
|
||||
}
|
||||
@@ -52,8 +52,17 @@ jobs:
|
||||
# END REPOSITORY-SPECIFIC BUILD
|
||||
|
||||
- name: Verify website in Chromium
|
||||
env:
|
||||
ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/site-browser
|
||||
run: |
|
||||
uv run --no-config --no-project --with playwright==1.63.0 playwright install --with-deps chromium
|
||||
# Resolve the same supported browser as captures; Playwright is the
|
||||
# driver only and must not provision a separate browser/runtime.
|
||||
ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
|
||||
CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
|
||||
uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
|
||||
CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
|
||||
export CHROME_BINARY
|
||||
test -x "$CHROME_BINARY"
|
||||
uv run --no-config --no-project --with playwright==1.63.0 python .github/pages/verify.py "$SITE_OUTPUT" --evidence "$RUNNER_TEMP/site-evidence"
|
||||
- name: Save website verification evidence
|
||||
if: always()
|
||||
|
||||
+119
-89
@@ -274,24 +274,21 @@ jobs:
|
||||
if: inputs.push_digests
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Verify OpenCode install with a read-only browser profile
|
||||
- name: Verify optional agent install with a read-only browser profile
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
PROFILE_DIR="$(mktemp -d)"
|
||||
# shellcheck disable=SC2016
|
||||
"$DOCKER_BINARY" run --rm --platform '${{ matrix.platform }}' \
|
||||
--volume "$PROFILE_DIR:/home/archivebox/.config/chromium:ro" \
|
||||
"$IMAGE" bash -c '
|
||||
set -Eeuo pipefail
|
||||
test -w "$XDG_CONFIG_HOME"
|
||||
test ! -w "$XDG_CONFIG_HOME/chromium"
|
||||
# Check the shipped executable before the installer can repair
|
||||
# an image that accidentally omitted this opt-in dependency.
|
||||
"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
|
||||
archivebox install opencode
|
||||
"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version
|
||||
uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_archivebox_install; verify_archivebox_install()"
|
||||
'
|
||||
|
||||
- name: Validate exact built image version and commit
|
||||
@@ -381,100 +378,133 @@ jobs:
|
||||
-u smbd -u nmbd -u samba-ad-dc -u nfs-server
|
||||
ps -eo pid,ppid,stat,etimes,wchan:32,comm
|
||||
|
||||
- name: Validate Docker UID and GID behavior on real NFS and SMB mounts
|
||||
if: inputs.full_tests && matrix.platform == 'linux/amd64'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
NFS_SOURCE="/srv/archivebox-ci-nfs"
|
||||
SMB_SOURCE="/srv/archivebox-ci-smb"
|
||||
NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
|
||||
SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
|
||||
mounted_nfs=0
|
||||
mounted_smb=0
|
||||
|
||||
cleanup() {
|
||||
local original_status=$? cleanup_status=0
|
||||
trap - EXIT
|
||||
if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$original_status" != "0" ]]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
exit "$cleanup_status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
|
||||
"$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
|
||||
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet smbd.service
|
||||
|
||||
"$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
|
||||
mounted_nfs=1
|
||||
"$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
|
||||
-o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
|
||||
mounted_smb=1
|
||||
|
||||
NFS_TEST_DIR="$NFS_MOUNT" \
|
||||
SMB_TEST_DIR="$SMB_MOUNT" \
|
||||
IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
|
||||
"$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
|
||||
|
||||
- name: Validate Docker port compatibility
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
|
||||
|
||||
- name: Validate Docker CLI add/list and compose behavior
|
||||
- name: Validate Docker mounts, ports, CLI and compose concurrently
|
||||
if: inputs.full_tests && matrix.platform == 'linux/amd64'
|
||||
shell: bash
|
||||
env:
|
||||
ARCHIVEBOX_IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
"$MKDIR_BINARY" "${{ github.workspace }}/data"
|
||||
"$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
|
||||
# These checks own separate collections, Compose projects and ports.
|
||||
# Subshells keep mount cleanup traps and environment changes isolated.
|
||||
validate_mounts() (
|
||||
set -Eeuo pipefail
|
||||
NFS_SOURCE="/srv/archivebox-ci-nfs"
|
||||
SMB_SOURCE="/srv/archivebox-ci-smb"
|
||||
NFS_MOUNT="$RUNNER_TEMP/archivebox-nfs-mount"
|
||||
SMB_MOUNT="$RUNNER_TEMP/archivebox-smb-mount"
|
||||
mounted_nfs=0
|
||||
mounted_smb=0
|
||||
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
|
||||
"$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
|
||||
# Invoked by the subshell EXIT trap.
|
||||
# shellcheck disable=SC2329
|
||||
cleanup() {
|
||||
local original_status=$? cleanup_status=0
|
||||
trap - EXIT
|
||||
if [[ "$mounted_smb" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$SMB_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$mounted_nfs" == "1" ]] && ! "$SUDO_BINARY" "$UMOUNT_BINARY" "$NFS_MOUNT"; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if ! "$SUDO_BINARY" "$SYSTEMCTL_BINARY" stop smbd.service nfs-kernel-server.service; then
|
||||
cleanup_status=1
|
||||
fi
|
||||
if [[ "$original_status" != "0" ]]; then
|
||||
exit "$original_status"
|
||||
fi
|
||||
exit "$cleanup_status"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
"$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
|
||||
echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$MKDIR_BINARY" -p /etc/exports.d
|
||||
"$SUDO_BINARY" "$CHMOD_BINARY" 0777 "$NFS_SOURCE" "$SMB_SOURCE" "$NFS_MOUNT" "$SMB_MOUNT"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '%s *(rw,sync,no_subtree_check,root_squash,insecure)\\n' '$NFS_SOURCE' > /etc/exports.d/archivebox-ci.exports"
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c \
|
||||
"printf '\\n[archivebox-ci]\\npath = %s\\nbrowseable = yes\\nread only = no\\nguest ok = yes\\nguest only = yes\\nforce user = nobody\\ncreate mask = 0777\\ndirectory mask = 0777\\n' '$SMB_SOURCE' >> /etc/samba/smb.conf"
|
||||
|
||||
archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
|
||||
[[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
[[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$EXPORTFS_BINARY" -ra
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet nfs-kernel-server.service
|
||||
"$SUDO_BINARY" "$SYSTEMCTL_BINARY" restart smbd.service
|
||||
"$SYSTEMCTL_BINARY" is-active --quiet smbd.service
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox init
|
||||
"$DOCKER_BINARY" compose up -d --wait
|
||||
"$DOCKER_BINARY" network connect archivebox_default www-nginx
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
|
||||
[[ "$index_html" == *ArchiveBox* ]]
|
||||
[[ "$admin_js" == *window.django* ]]
|
||||
"$SUDO_BINARY" "$MOUNT_NFS_BINARY" -o vers=3,nolock "127.0.0.1:$NFS_SOURCE" "$NFS_MOUNT"
|
||||
mounted_nfs=1
|
||||
"$SUDO_BINARY" "$MOUNT_CIFS_BINARY" //127.0.0.1/archivebox-ci "$SMB_MOUNT" \
|
||||
-o "guest,vers=3.0,uid=0,gid=0,forceuid,forcegid,file_mode=0777,dir_mode=0777,nounix"
|
||||
mounted_smb=1
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
[[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
|
||||
"$DOCKER_BINARY" compose down
|
||||
NFS_TEST_DIR="$NFS_MOUNT" \
|
||||
SMB_TEST_DIR="$SMB_MOUNT" \
|
||||
IMAGE="${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}" \
|
||||
"$BASH_BINARY" bin/validate_docker_uid_gid.sh --local-only
|
||||
)
|
||||
validate_cli() (
|
||||
set -Eeuo pipefail
|
||||
"$MKDIR_BINARY" "${{ github.workspace }}/data"
|
||||
"$DOCKER_BINARY" run -v "${{ github.workspace }}/data":/data "$ARCHIVEBOX_IMAGE" init
|
||||
|
||||
"$SUDO_BINARY" "$BASH_BINARY" -c 'echo "127.0.0.1 www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
|
||||
"$DOCKER_BINARY" run --name www-nginx -p 80:80 -d nginx:1.29.5@sha256:0236ee02dcbce00b9bd83e0f5fbc51069e7e1161bd59d99885b3ae1734f3392e
|
||||
|
||||
"$DOCKER_BINARY" run -v "$PWD"/data:/data --network host "$ARCHIVEBOX_IMAGE" add http://www.test-nginx-1.local
|
||||
# The argument form above exercises the complete default pipeline.
|
||||
# Keep a real capture for stdin parsing without repeating every plugin.
|
||||
echo "http://www.test-nginx-2.local" | "$DOCKER_BINARY" run -i --network host -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" add --plugins=wget,hashes
|
||||
|
||||
archive_list="$("$DOCKER_BINARY" run -v "$PWD"/data:/data "$ARCHIVEBOX_IMAGE" list)"
|
||||
[[ "$archive_list" == *"www.test-nginx-1.local"* ]] || { echo "The site 1 isn't in the list"; exit 1; }
|
||||
[[ "$archive_list" == *"www.test-nginx-2.local"* ]] || { echo "The site 2 isn't in the list"; exit 1; }
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox init
|
||||
"$DOCKER_BINARY" compose up -d --wait
|
||||
"$DOCKER_BINARY" network connect archivebox_default www-nginx
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
admin_js="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/static/admin/js/jquery.init.js')"
|
||||
[[ "$index_html" == *ArchiveBox* ]]
|
||||
[[ "$admin_js" == *window.django* ]]
|
||||
|
||||
"$DOCKER_BINARY" compose run --rm archivebox add --plugins=parse_txt_urls 'http://www-nginx/#test_docker'
|
||||
index_html="$("$CURL_BINARY" --silent --location 'http://127.0.0.1:5797/public/')"
|
||||
[[ "$index_html" == *'http://www-nginx/#test_docker'* ]]
|
||||
"$DOCKER_BINARY" compose down
|
||||
)
|
||||
validate_ports() (
|
||||
IMAGE="$ARCHIVEBOX_IMAGE" "$BASH_BINARY" bin/validate_docker_ports.sh
|
||||
)
|
||||
|
||||
checks=(mounts ports cli)
|
||||
pids=()
|
||||
validate_mounts > "$RUNNER_TEMP/docker-mounts.log" 2>&1 &
|
||||
pids+=("$!")
|
||||
validate_ports > "$RUNNER_TEMP/docker-ports.log" 2>&1 &
|
||||
pids+=("$!")
|
||||
validate_cli > "$RUNNER_TEMP/docker-cli.log" 2>&1 &
|
||||
pids+=("$!")
|
||||
status=0
|
||||
for index in "${!checks[@]}"; do
|
||||
check="${checks[$index]}"
|
||||
echo "::group::Docker $check validation"
|
||||
if wait "${pids[$index]}"; then
|
||||
echo "PASS: Docker $check validation"
|
||||
else
|
||||
echo "::error::Docker $check validation failed"
|
||||
status=1
|
||||
fi
|
||||
cat "$RUNNER_TEMP/docker-$check.log"
|
||||
echo "::endgroup::"
|
||||
done
|
||||
exit "$status"
|
||||
|
||||
- name: Validate Docker port compatibility
|
||||
if: ${{ !(inputs.full_tests && matrix.platform == 'linux/amd64') }}
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.push_digests && format('{0}@{1}', env.DOCKERHUB_IMAGE, steps.docker_build.outputs.digest) || matrix.local_tag }}
|
||||
run: '"$BASH_BINARY" bin/validate_docker_ports.sh'
|
||||
|
||||
- name: Validate compressed image size
|
||||
if: inputs.full_tests && !inputs.push_digests
|
||||
|
||||
@@ -392,7 +392,8 @@ jobs:
|
||||
|
||||
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
with:
|
||||
install: true
|
||||
# These images were already built and tested; only publish manifests.
|
||||
driver: docker
|
||||
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
@@ -525,17 +526,27 @@ jobs:
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${GHCR_IMAGE}@sha256:${digest}"); done
|
||||
$DOCKER_BINARY buildx imagetools create "${TAG_ARGS[@]}" "${REFS[@]}"
|
||||
|
||||
- name: Verify published Docker images run
|
||||
- name: Verify published tags contain the exact tested images
|
||||
env:
|
||||
DOCKERHUB_TAGS: ${{ steps.docker_meta.outputs.dockerhub_tags }}
|
||||
GHCR_TAGS: ${{ steps.docker_meta.outputs.ghcr_tags }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
VERSION='${{ needs.python-release.outputs.version }}'
|
||||
SHORT_SHA="${RELEASE_SHA::12}"
|
||||
for image in "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}" "${GHCR_IMAGE}:sha-${SHORT_SHA}"; do
|
||||
"$DOCKER_BINARY" buildx imagetools inspect "$image" >/dev/null
|
||||
INSTALLED_VERSION="$("$DOCKER_BINARY" run --rm "$image" --version)"
|
||||
test "$INSTALLED_VERSION" = "$VERSION"
|
||||
done
|
||||
# Both architectures already passed CLI/version/capture acceptance by
|
||||
# digest. Verify promotion preserves those exact images and attestations
|
||||
# instead of downloading and running the same image again on one arch.
|
||||
mapfile -t DIGESTS < <($UV_BINARY run --no-cache --no-project python -c 'import re; from pathlib import Path; print("\n".join(sorted(path.name for path in Path("/tmp/digests").iterdir() if path.is_file() and re.fullmatch(r"[0-9a-f]{64}", path.name))))')
|
||||
REFS=()
|
||||
for digest in "${DIGESTS[@]}"; do REFS+=("${DOCKERHUB_IMAGE}@sha256:${digest}"); done
|
||||
EXPECTED="$("$DOCKER_BINARY" buildx imagetools create --dry-run "${REFS[@]}" | "$JQ_BINARY" -ceS '.manifests | sort_by(.digest)')"
|
||||
"$JQ_BINARY" -e 'map(select(.platform.os == "linux") | .platform.architecture) | sort == ["amd64", "arm64"]' <<< "$EXPECTED"
|
||||
while IFS= read -r image; do
|
||||
[[ -n "$image" ]] || continue
|
||||
ACTUAL="$("$DOCKER_BINARY" buildx imagetools inspect --raw "$image" | "$JQ_BINARY" -ceS 'select(.schemaVersion == 2) | .manifests | sort_by(.digest)')"
|
||||
[[ "$ACTUAL" == "$EXPECTED" ]] || { echo "Published tag differs from tested images: $image" >&2; exit 1; }
|
||||
echo "Verified exact tested images: $image"
|
||||
done < <(printf '%s\n' "$DOCKERHUB_TAGS" "$GHCR_TAGS")
|
||||
|
||||
- name: Record published image source
|
||||
shell: bash
|
||||
|
||||
@@ -84,12 +84,20 @@ jobs:
|
||||
env:
|
||||
STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
STAGING_KNOWN_HOSTS: ${{ secrets.STAGING_KNOWN_HOSTS }}
|
||||
ABXPKG_LIB_DIR: ${{ runner.tool_cache }}/abxpkg/staging-browser
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
printf '%s\n' "$STAGING_SSH_KEY" > "$RUNNER_TEMP/staging-key"
|
||||
printf '%s\n' "$STAGING_KNOWN_HOSTS" > "$RUNNER_TEMP/staging-known-hosts"
|
||||
uv run --no-project --with playwright==1.63.0 playwright install chromium
|
||||
# Use the candidate's normal provider/cache path, not a second
|
||||
# Playwright-managed browser that bypasses abxpkg validation.
|
||||
ABX_DL_VERSION="$(uv run --no-project python -c 'import tomllib; print(next(p["version"] for p in tomllib.load(open("uv.lock", "rb"))["package"] if p["name"] == "abx-dl"))')"
|
||||
CHROME_SCHEMA="$(uv run --no-project --with "abx-dl==$ABX_DL_VERSION" python -c 'from abx_plugins import get_plugins_dir; print(get_plugins_dir() / "chrome" / "config.json")')"
|
||||
uv run --no-project --with "abx-dl==$ABX_DL_VERSION" abxpkg env --install --json --deps-from="$CHROME_SCHEMA:required_binaries" > "$RUNNER_TEMP/browser-env.json"
|
||||
CHROME_BINARY="$(uv run --no-project python -c 'import json, sys; print(json.load(open(sys.argv[1]))["CHROME_BINARY"])' "$RUNNER_TEMP/browser-env.json")"
|
||||
test -x "$CHROME_BINARY"
|
||||
echo "CHROME_BINARY=$CHROME_BINARY" >> "$GITHUB_ENV"
|
||||
- name: Verify actual deployment, full captures and rendered snapshots
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -112,8 +112,21 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set job-local runtime paths
|
||||
env:
|
||||
CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
|
||||
run: |
|
||||
workers=2
|
||||
cache_suffix=''
|
||||
if [[ "$CI_RUNNER_ENVIRONMENT" == github-hosted ]]; then
|
||||
workers="$(nproc)"
|
||||
cache_suffix="-workers-$workers"
|
||||
# Avoid the hosted image's cold Chromium executable page faults.
|
||||
# abxpkg owns installation and reuse in the existing runtime cache.
|
||||
echo "CHROME_BINPROVIDERS=playwright,puppeteer" >> "$GITHUB_ENV"
|
||||
fi
|
||||
{
|
||||
echo "CI_TEST_WORKERS=$workers"
|
||||
echo "ABXPKG_CACHE_WORKERS_SUFFIX=$cache_suffix"
|
||||
echo "ABXPKG_LIB_DIR=$RUNNER_TEMP/archivebox-lib"
|
||||
echo "DATA_DIR=$RUNNER_TEMP/archivebox-test-data"
|
||||
echo "PERSONAS_DIR=$RUNNER_TEMP/abx-personas"
|
||||
@@ -129,6 +142,7 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ env.CI_PYTHON_VERSION }}
|
||||
architecture: x64
|
||||
id: python
|
||||
|
||||
- uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6
|
||||
with:
|
||||
@@ -136,6 +150,41 @@ jobs:
|
||||
enable-cache: false
|
||||
cache-dependency-glob: uv.lock
|
||||
|
||||
# Self-hosted tool storage survives jobs. Normal abxpkg validation and
|
||||
# install locks allow reuse across files; cold-install tests use tmp_path.
|
||||
- name: Reuse persistent runner tools
|
||||
if: runner.environment == 'self-hosted'
|
||||
env:
|
||||
PERSISTENT_ABXPKG_LIB: ${{ runner.tool_cache }}/abxpkg/${{ github.repository }}/${{ steps.python.outputs.python-version }}
|
||||
run: |
|
||||
mkdir -p "$PERSISTENT_ABXPKG_LIB"
|
||||
echo "ABXPKG_LIB_DIR=$PERSISTENT_ABXPKG_LIB" >> "$GITHUB_ENV"
|
||||
|
||||
# Restored binaries still pass normal abxpkg resolution/version checks.
|
||||
# Installation and isolation tests explicitly choose fresh temporary libs.
|
||||
- name: Identify runtime dependency cache
|
||||
shell: bash
|
||||
run: |
|
||||
uv run --no-project python - <<'PYTHON' >> "$GITHUB_ENV"
|
||||
import hashlib
|
||||
import json
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
packages = tomllib.loads(Path("uv.lock").read_text())["package"]
|
||||
versions = {p["name"]: p["version"] for p in packages if p["name"] in {"abxpkg", "abx-plugins"} and "editable" not in p["source"]}
|
||||
print("ABXPKG_CACHE_DEPS=" + hashlib.sha256(json.dumps(versions, sort_keys=True).encode()).hexdigest())
|
||||
PYTHON
|
||||
|
||||
- name: Reuse validated runtime dependencies
|
||||
if: runner.environment != 'self-hosted'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
|
||||
with:
|
||||
path: ${{ runner.temp }}/archivebox-lib
|
||||
key: abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-${{ env.ABXPKG_CACHE_DEPS }}${{ env.ABXPKG_CACHE_WORKERS_SUFFIX }}
|
||||
restore-keys: |
|
||||
abxpkg-v1-${{ runner.os }}-${{ runner.arch }}-${{ runner.environment }}-${{ steps.python.outputs.python-version }}-${{ matrix.test.name }}-
|
||||
|
||||
- name: Resolve optional build dependencies through abxpkg
|
||||
if: matrix.test.extra != '' || contains(matrix.test.paths_arg, 'test_postgres_backend')
|
||||
env:
|
||||
@@ -169,35 +218,25 @@ jobs:
|
||||
extra_args=(--extra "$TEST_EXTRA")
|
||||
fi
|
||||
uv sync --no-cache --locked --dev "${extra_args[@]}"
|
||||
- name: Install Playwright Chromium for browser tests
|
||||
if: contains(matrix.test.paths_arg, 'test_proof_previews') || contains(matrix.test.paths_arg, 'test_snapshot_output_stacks') || contains(matrix.test.paths_arg, 'test_ui_admin_snapshot') || contains(matrix.test.paths_arg, 'test_cli_extract')
|
||||
run: uv run --no-cache --no-sync --no-sources playwright install chromium
|
||||
|
||||
- name: Run ${{ matrix.test.name }}
|
||||
env:
|
||||
TEST_PATHS: ${{ matrix.test.paths_arg }}
|
||||
CI_RUNNER_ENVIRONMENT: ${{ runner.environment }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
echo "Running pytest with $CI_TEST_WORKERS workers on $CI_RUNNER_ENVIRONMENT"
|
||||
read -r -a test_paths <<< "$TEST_PATHS"
|
||||
test_args=()
|
||||
for test_path in "${test_paths[@]}"; do
|
||||
test -f "$test_path"
|
||||
test_args+=("$GITHUB_WORKSPACE/$test_path")
|
||||
done
|
||||
test_status=0
|
||||
for test_path in "${test_paths[@]}"; do
|
||||
test_name="$(basename "$test_path" .py)"
|
||||
test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
|
||||
export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
|
||||
export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
|
||||
export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
|
||||
if (( ${#test_paths[@]} > 1 )); then
|
||||
export ABXPKG_LIB_DIR="$RUNNER_TEMP/archivebox-lib/$test_slug"
|
||||
fi
|
||||
mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
|
||||
uv run --no-cache --no-sync --no-sources pytest -vs -n 2 "$GITHUB_WORKSPACE/$test_path" --durations=20 \
|
||||
--basetemp="$RUNNER_TEMP/pytest/$test_slug" \
|
||||
--junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
|
||||
done
|
||||
exit "$test_status"
|
||||
# Each test still gets isolated DB/filesystem state from conftest.
|
||||
# Keep one worker pool per batch instead of starting it for every file.
|
||||
mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest"
|
||||
uv run --no-cache --no-sync --no-sources pytest -vs -n "$CI_TEST_WORKERS" "${test_args[@]}" --durations=20 \
|
||||
--basetemp="$RUNNER_TEMP/pytest" \
|
||||
--junitxml="$RUNNER_TEMP/test-results/results.xml"
|
||||
- name: Save test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
|
||||
+24
-7
@@ -3,14 +3,14 @@
|
||||
# Multistage ArchiveBox Dockerfile that consumes the abx-dl runtime image.
|
||||
# WHY: abx-dl owns Python, Node, Chromium, and extraction dependencies so the
|
||||
# standalone downloader stays independent of ArchiveBox's server features.
|
||||
# ArchiveBox-only dependencies (Sonic, OpenCode, supervisor, Django) belong in
|
||||
# ArchiveBox-only dependencies (server plugins, supervisor, Django) belong in
|
||||
# the layers added by this Dockerfile, never in the shared abx-dl base image.
|
||||
# REQUIRED INSTALLATION CONTRACT — do not work around failures:
|
||||
# 1. IMAGE CONSTRUCTION: preinstall ALL plugins supported by runtime="archivebox"
|
||||
# and ALL their required_binaries, regardless of default enabled state.
|
||||
# Inherit the complete downloader dependencies from the tested abx-dl image;
|
||||
# install every remaining ArchiveBox-only dependency in this image's layers,
|
||||
# including OpenCode and Sonic. Never put server dependencies in abx-dl.
|
||||
# including optional server plugins. Never put server dependencies in abx-dl.
|
||||
# Use the normal plugin dependency pipeline: install command -> config.json
|
||||
# required_binaries -> abxpkg. Never add manual apt/pip/npm installs to hide
|
||||
# failures in that pipeline. OS/interpreter bootstrap is a separate concern,
|
||||
@@ -156,6 +156,22 @@ apt-get install -qq -y --no-install-recommends x11-utils
|
||||
--no-install-package abx-plugins \
|
||||
--no-install-package abx-dl \
|
||||
--no-sources
|
||||
# Sync may replace inherited dependencies (e.g. pydantic/rich-click), removing
|
||||
# their bytecode. Runtime writes are disabled, so missing caches would make
|
||||
# every hook compile them again. Compile only missing entries: compileall's
|
||||
# timestamp fast path rewrites even valid checked-hash caches, copying the
|
||||
# entire inherited venv into this OCI layer for no runtime benefit.
|
||||
/usr/bin/uv run --no-project --no-sync python - <<'PY'
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import py_compile
|
||||
import sysconfig
|
||||
|
||||
for source in Path(sysconfig.get_path("purelib")).rglob("*.py"):
|
||||
cache = Path(importlib.util.cache_from_source(str(source)))
|
||||
if not cache.is_file():
|
||||
py_compile.compile(str(source), doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
|
||||
PY
|
||||
builder_abxpkg_lib_dir=/tmp/archivebox-builder-abxpkg
|
||||
ABXPKG_NO_CACHE=True abxpkg env --install --binproviders=env,apt --lib="$builder_abxpkg_lib_dir" --overrides='{"apt":{"install_args":["binutils"]}}' strip >/dev/null
|
||||
/usr/bin/find /venv/lib/python3.*/site-packages -type f -name '*.so' -print0 > /tmp/archivebox-native-libraries
|
||||
@@ -250,15 +266,13 @@ RUN --mount=type=cache,target=/opt/archivebox/lib/cache,sharing=locked,mode=1777
|
||||
|
||||
# Install the complete ArchiveBox catalog, including disabled server plugins.
|
||||
# Explicit plugin names make installation independent of enabled defaults.
|
||||
RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-opencode-$TARGETARCH \
|
||||
RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=archivebox-server-plugins-$TARGETARCH \
|
||||
chown -R "$DEFAULT_ARCHIVEBOX_UID:$DEFAULT_ARCHIVEBOX_GID" /var/tmp/abxpkg-cache \
|
||||
&& chmod 1777 /var/tmp/abxpkg-cache \
|
||||
&& export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
|
||||
&& env ABXPKG_TMP_CACHE_DIR=/var/tmp/abxpkg-cache XDG_CACHE_HOME=/var/tmp/abxpkg-cache \
|
||||
setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups abx-dl install $ABX_DOCKER_PLUGINS \
|
||||
# pnpm includes musl variants that Debian's glibc runtime cannot use.
|
||||
&& find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules" -type l -name 'opencode-linux-*-musl' -delete \
|
||||
&& find "$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.pnpm" -maxdepth 1 -type d -name 'opencode-linux-*-musl@*' -exec rm -rf {} + \
|
||||
&& UV_NO_CACHE=true /usr/bin/uv run --no-project python -c 'from abx_plugins.plugins.opencode.image import prune_incompatible_image_files; prune_incompatible_image_files()' \
|
||||
# pnpm's explicit store lives under ABXPKG_LIB_DIR, regardless of XDG_CACHE_HOME.
|
||||
# Retaining its download copies made image size depend on whether the build
|
||||
# preserved hardlinks to installed files. Remove the store in this layer;
|
||||
@@ -267,7 +281,7 @@ RUN --mount=type=cache,target=/var/tmp/abxpkg-cache,sharing=locked,mode=1777,id=
|
||||
&& rm -rf "$ABXPKG_LIB_DIR/cache/pnpm"
|
||||
RUN --network=none export ABX_DOCKER_PLUGINS="$(/venv/bin/python3 -c 'from abx_dl.catalog import PluginCatalog; print(" ".join(PluginCatalog.discover(runtime="archivebox")))')" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups \
|
||||
bash -c '"$ABXPKG_LIB_DIR/pnpm/packages/opencode/node_modules/.bin/opencode" --version && abx-dl install $ABX_DOCKER_PLUGINS'
|
||||
bash -c 'uv run --no-project python -c "from abx_plugins.plugins.opencode.image import verify_installed; verify_installed()" && abx-dl install $ABX_DOCKER_PLUGINS'
|
||||
|
||||
WORKDIR "$DATA_DIR"
|
||||
RUN echo "[+] Initializing image collection..." \
|
||||
@@ -287,6 +301,9 @@ RUN chmod +x "$CODE_DIR"/bin/*.sh \
|
||||
&& chmod g+w "$TMP_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH"
|
||||
|
||||
RUN for forbidden_bin in gcc g++ make; do ! abxpkg load --binproviders=env "$forbidden_bin" >/dev/null 2>&1 || (echo "Unexpected build tool in runtime: $forbidden_bin" >&2 && exit 1); done \
|
||||
&& test -f "$(/venv/bin/python -c 'import pydantic; print(pydantic.__cached__)')" \
|
||||
&& test -f "$(/venv/bin/python -c 'import rich_click; print(rich_click.__cached__)')" \
|
||||
&& test -f "$(/venv/bin/python -c 'import platformdirs; print(platformdirs.__cached__)')" \
|
||||
&& stat -c "%U:%G %a %n" "$CONFIG_DIR" "$ABXPKG_LIB_DIR" "$PLAYWRIGHT_BROWSERS_PATH" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$CONFIG_DIR" \
|
||||
&& setpriv --reuid="$ARCHIVEBOX_USER" --regid="$ARCHIVEBOX_USER" --init-groups test -w "$ABXPKG_LIB_DIR" \
|
||||
|
||||
+44
-17
@@ -55,7 +55,6 @@ from archivebox.api.v1_crawls import CrawlSchema, get_crawl_by_ref
|
||||
from archivebox.search.config import get_search_mode, get_search_mode_backend
|
||||
from archivebox.search.query import apply_snapshot_search
|
||||
from archivebox.core.snapshot_status import filter_snapshots_by_status, normalize_snapshot_status
|
||||
from archivebox.workers.models import RETRY_AT_MAX
|
||||
|
||||
|
||||
router = Router(tags=["Core Models"])
|
||||
@@ -212,11 +211,22 @@ class ArchiveResultFilterSchema(FilterSchema):
|
||||
created_at__lt: Annotated[datetime | None, FilterLookup("created_at__lt")] = None
|
||||
|
||||
|
||||
def _with_archiveresult_processes(queryset):
|
||||
# cmd/pwd/version delegate to Process and Binary. Without this join a full
|
||||
# capture's metadata response performs two extra indexed lookups per hook
|
||||
# (roughly 100 queries), despite the initial Snapshot UUID lookup being fast.
|
||||
# Logs and the hydrated hook environment are not response fields; loading
|
||||
# those potentially large values would replace round trips with excess I/O.
|
||||
return queryset.select_related("process__binary").defer("process__stdout", "process__stderr", "process__env")
|
||||
|
||||
|
||||
@router.get("/archiveresults", response=list[ArchiveResultSchema], url_name="get_archiveresult")
|
||||
@paginate(CustomPagination)
|
||||
def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterSchema]):
|
||||
"""List all ArchiveResult entries matching these filters."""
|
||||
queryset = filters.filter(ArchiveResult.objects.all())
|
||||
queryset = filters.filter(
|
||||
_with_archiveresult_processes(ArchiveResult.objects.all()).prefetch_related("snapshot__crawl__created_by", "snapshot__tags"),
|
||||
)
|
||||
if filters.search or filters.snapshot_tag:
|
||||
return queryset.distinct()
|
||||
return queryset
|
||||
@@ -224,26 +234,31 @@ def get_archiveresults(request: HttpRequest, filters: Query[ArchiveResultFilterS
|
||||
|
||||
def _uuid_ref_query(field_name: str, ref: str) -> Q:
|
||||
raw_ref = str(ref or "").strip()
|
||||
query = Q(**{f"{field_name}__startswith": raw_ref})
|
||||
if raw_ref:
|
||||
query |= Q(**{f"{field_name}__icontains": raw_ref})
|
||||
try:
|
||||
parsed_uuid = UUID(raw_ref)
|
||||
except (TypeError, ValueError):
|
||||
query = Q(**{f"{field_name}__startswith": raw_ref})
|
||||
if raw_ref:
|
||||
query |= Q(**{f"{field_name}__icontains": raw_ref})
|
||||
normalized_ref = raw_ref.replace("-", "")
|
||||
if normalized_ref and normalized_ref != raw_ref:
|
||||
query |= Q(**{f"{field_name}__startswith": normalized_ref})
|
||||
query |= Q(**{f"{field_name}__icontains": normalized_ref})
|
||||
else:
|
||||
query |= Q(**{field_name: parsed_uuid})
|
||||
query |= Q(**{f"{field_name}__startswith": parsed_uuid.hex})
|
||||
return query
|
||||
return query
|
||||
# An exact UUID cannot need a substring fallback. OR-ing it with LIKE
|
||||
# prevents an indexed lookup and made Cabbage metadata reads take seconds.
|
||||
return Q(**{field_name: parsed_uuid})
|
||||
|
||||
|
||||
@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
|
||||
def get_archiveresult(request: HttpRequest, archiveresult_id: str):
|
||||
"""Get a specific ArchiveResult by id."""
|
||||
return ArchiveResult.objects.get(_uuid_ref_query("id", archiveresult_id))
|
||||
return (
|
||||
_with_archiveresult_processes(ArchiveResult.objects.all())
|
||||
.select_related("snapshot__crawl__created_by")
|
||||
.prefetch_related("snapshot__tags")
|
||||
.get(_uuid_ref_query("id", archiveresult_id))
|
||||
)
|
||||
|
||||
|
||||
def _normalize_uploaded_archiveresult_plugin(plugin: str) -> str:
|
||||
@@ -351,6 +366,12 @@ def _summarize_archiveresult_output_files(output_files: dict[str, dict[str, Any]
|
||||
|
||||
def _get_snapshot_by_ref(snapshot_id: str):
|
||||
queryset = Snapshot.objects.select_related("crawl__created_by")
|
||||
try:
|
||||
full_id = UUID(snapshot_id)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
else:
|
||||
return queryset.get(pk=full_id)
|
||||
try:
|
||||
return queryset.get(_uuid_ref_query("id", snapshot_id) | Q(timestamp__startswith=snapshot_id))
|
||||
except Snapshot.DoesNotExist:
|
||||
@@ -632,6 +653,7 @@ class SnapshotSchema(Schema):
|
||||
TYPE: str = "core.models.Snapshot"
|
||||
id: UUID
|
||||
crawl_id: UUID
|
||||
persona: str | None
|
||||
created_by_id: str
|
||||
created_by_username: str
|
||||
created_at: datetime
|
||||
@@ -654,6 +676,13 @@ class SnapshotSchema(Schema):
|
||||
def resolve_archive_path(obj):
|
||||
return obj.archive_path_from_db
|
||||
|
||||
@staticmethod
|
||||
def resolve_persona(obj):
|
||||
# A deduplicated submission can reuse a capture from a different
|
||||
# persona. Report its actual DB-backed owner, not the new request's choice.
|
||||
persona = obj.crawl.resolve_persona()
|
||||
return persona.name if persona else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_created_by_id(obj):
|
||||
return str(obj.created_by.pk)
|
||||
@@ -676,12 +705,14 @@ class SnapshotSchema(Schema):
|
||||
|
||||
@staticmethod
|
||||
def resolve_num_archiveresults(obj, context):
|
||||
return obj.archiveresult_set.all().distinct().count()
|
||||
# This reverse FK contains each result once. DISTINCT would materialize
|
||||
# wide JSON output columns just to count rows through snapshot_id's index.
|
||||
return obj.archiveresult_set.count()
|
||||
|
||||
@staticmethod
|
||||
def resolve_archiveresults(obj, context):
|
||||
if bool(context["request"].__dict__.get("with_archiveresults", False)):
|
||||
return obj.archiveresult_set.all().distinct()
|
||||
return _with_archiveresult_processes(obj.archiveresult_set.all())
|
||||
return ArchiveResult.objects.none()
|
||||
|
||||
|
||||
@@ -1058,11 +1089,7 @@ def delete_snapshot(request: HttpRequest, snapshot_id: str):
|
||||
crawl_id_str = str(snapshot.crawl_id)
|
||||
# Use the same durable queue as the admin delete action. Stopping writers
|
||||
# and removing archive files belongs to the runner, never the HTTP request.
|
||||
queued_count = Snapshot.objects.filter(pk=snapshot.pk).update(
|
||||
status=Snapshot.DELETING_STATE,
|
||||
retry_at=RETRY_AT_MAX,
|
||||
modified_at=timezone.now(),
|
||||
)
|
||||
queued_count = Snapshot.objects.filter(pk=snapshot.pk).request_delete()
|
||||
return {
|
||||
"success": True,
|
||||
"snapshot_id": snapshot_id_str,
|
||||
|
||||
@@ -102,7 +102,14 @@ class PagedQuerySet(models.QuerySet):
|
||||
prefix = {ordered_field_names[i]: last_values[i] for i in range(idx)}
|
||||
comparison = "lt" if descending else "gt"
|
||||
page_filter |= models.Q(**prefix, **{f"{field_name}__{comparison}": last_values[idx]})
|
||||
batch_qs = batch_qs.filter(page_filter)
|
||||
# Anchor the OR tie-breaks with an explicit leading range so
|
||||
# SQLite/PG can seek the existing order index instead of
|
||||
# rescanning its prefix for every keyset page.
|
||||
first_comparison = "lte" if ordering[0].startswith("-") else "gte"
|
||||
batch_qs = batch_qs.filter(
|
||||
page_filter,
|
||||
**{f"{ordered_field_names[0]}__{first_comparison}": last_values[0]},
|
||||
)
|
||||
|
||||
batch_rows = list(batch_qs.values_list(*value_field_names)[:chunk_size])
|
||||
if not batch_rows:
|
||||
@@ -289,9 +296,21 @@ class ModelWithDeleteAfter(models.Model):
|
||||
return cls.objects.none()
|
||||
|
||||
@classmethod
|
||||
def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True) -> int:
|
||||
def delete_expired(cls, *, batch_size: int = 100, backfill_missing: bool = True, backfill_cursors: dict | None = None) -> int:
|
||||
if backfill_missing:
|
||||
missing_delete_at = list(cls.missing_delete_at_candidates().order_by("created_at", "pk")[:batch_size])
|
||||
# Bound the scanned rows BEFORE evaluating JSON policy/parent
|
||||
# joins. LIMIT on the policy query only bounded its output: finding
|
||||
# no matches could scan the entire history with a SQLite read lock.
|
||||
# The runner keeps a keyset cursor between idle passes, so disabled
|
||||
# policies cannot starve older rows. Restarting safely repeats work.
|
||||
page = cls.objects.order_by("-pk")
|
||||
before = backfill_cursors.get(cls) if backfill_cursors is not None else None
|
||||
if before is not None:
|
||||
page = page.filter(pk__lt=before)
|
||||
page_ids = list(page.values_list("pk", flat=True)[:batch_size])
|
||||
if backfill_cursors is not None:
|
||||
backfill_cursors[cls] = page_ids[-1] if len(page_ids) == batch_size else None
|
||||
missing_delete_at = list(cls.missing_delete_at_candidates().filter(pk__in=page_ids).order_by())
|
||||
for obj in missing_delete_at:
|
||||
if obj.set_delete_at_from_config():
|
||||
cls.objects.filter(pk=obj.pk, delete_at__isnull=True).update(
|
||||
|
||||
@@ -423,7 +423,23 @@ def main(**kwargs):
|
||||
existing = [token.strip() for token in (kwargs.get("plugins") or "").split(",") if token.strip()]
|
||||
kwargs["plugins"] = ",".join(dict.fromkeys([*existing, *selected]))
|
||||
|
||||
add(urls=urls, **kwargs)
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
# CLI commands own their process record and logs. Queue model saves
|
||||
# must not register a process implicitly: HTTP add() shares that path.
|
||||
process = Process.current()
|
||||
exit_code = 0
|
||||
try:
|
||||
add(urls=urls, **kwargs)
|
||||
except BaseException:
|
||||
exit_code = 1
|
||||
raise
|
||||
finally:
|
||||
# --bg/--index-only return without the foreground runner's cleanup.
|
||||
# Leaving them RUNNING makes the active-process index accumulate
|
||||
# historical CLI commands forever, defeating bounded recovery.
|
||||
if process.status == Process.StatusChoices.RUNNING:
|
||||
process.mark_exited(exit_code=exit_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -58,7 +58,6 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.machine.models import Binary, Machine, _canonical_binary_name
|
||||
from archivebox.services.runner import run_due_binary
|
||||
|
||||
machine = Machine.current()
|
||||
for requested_name in binary_names:
|
||||
@@ -80,7 +79,13 @@ def _install_raw_binary_names(binary_names: list[str], binproviders: str) -> Non
|
||||
binary.status = Binary.StatusChoices.QUEUED
|
||||
binary.retry_at = timezone.now()
|
||||
binary.save(update_fields=["binproviders", "overrides", "status", "retry_at", "modified_at"])
|
||||
run_due_binary(binary, lock_seconds=60)
|
||||
if binary.status == Binary.StatusChoices.INSTALLED:
|
||||
continue
|
||||
# The synchronous CLI needs the lifecycle's verified installation result,
|
||||
# not the scheduler's "claimed work" result. Reuse its failure/requeue and
|
||||
# health-stat handling so a failed install cannot exit successfully.
|
||||
if not binary.install_claimed(lock_seconds=60):
|
||||
raise click.ClickException(f"Binary {binary_name} is already being installed")
|
||||
|
||||
|
||||
def ensure_data_dir_lib_symlink(data_dir: Path, abxpkg_lib_dir: Path) -> Path | None:
|
||||
|
||||
@@ -298,7 +298,7 @@ def list_snapshots(
|
||||
Exit codes:
|
||||
0: Success (even if no results)
|
||||
"""
|
||||
from archivebox.misc.jsonl import write_record
|
||||
from archivebox.misc.jsonl import write_records
|
||||
|
||||
output_formats = sum(bool(output_format) for output_format in (csv, as_json, as_html))
|
||||
if output_formats > 1:
|
||||
@@ -351,9 +351,7 @@ def list_snapshots(
|
||||
return 0
|
||||
|
||||
if not is_tty:
|
||||
for snapshot_json in iter_snapshot_json(queryset):
|
||||
write_record(snapshot_json)
|
||||
count += 1
|
||||
count = write_records(iter_snapshot_json(queryset))
|
||||
rprint(f"[dim]Listed {count} snapshots[/dim]", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ def _binary_row_dedupe_key(
|
||||
def version(
|
||||
quiet: bool = False,
|
||||
binaries: Iterable[str] = (),
|
||||
no_cache: bool = False,
|
||||
) -> list[str]:
|
||||
"""Print the ArchiveBox version, debug metadata, and installed dependency versions"""
|
||||
|
||||
@@ -275,6 +276,7 @@ def version(
|
||||
prnt("", f"[yellow]Warning: Could not query collection machine config; resolving through abxpkg: {e}[/yellow]")
|
||||
|
||||
declared_binary_specs: dict[str, dict[str, object]] = {}
|
||||
binary_records_by_plugin: dict[str, list[dict[str, object]]] = {}
|
||||
for plugin_name, plugin in plugins.items():
|
||||
if not plugin_may_have_requested_binary(plugin):
|
||||
continue
|
||||
@@ -286,6 +288,10 @@ def version(
|
||||
derived_overrides=derived_config,
|
||||
run_output_dir=CONSTANTS.DATA_DIR,
|
||||
)
|
||||
# Render the exact requests resolved below. Rehydrating all schemas for
|
||||
# display doubled config work, and could describe different requests if
|
||||
# a config file changed while dependency resolution was running.
|
||||
binary_records_by_plugin[plugin_name] = binary_records
|
||||
for binary_record in binary_records:
|
||||
actual_name = str(binary_record["name"])
|
||||
logical_name = (
|
||||
@@ -296,9 +302,13 @@ def version(
|
||||
display_name = logical_name
|
||||
if not plugin_requested and not binary_is_requested(logical_name, actual_name, display_name):
|
||||
continue
|
||||
# This is a diagnostic: recheck the provider instead of trusting an
|
||||
# installation record or a cached version, including disabled plugins.
|
||||
binary_record["no_cache"] = True
|
||||
# Resolve through abxpkg, never the DB's last-known installation row.
|
||||
# Its cache validates current paths, permissions, package fingerprints
|
||||
# and runtime options. Forcing cold resolution here rehashed every
|
||||
# large binary and reran every version subprocess on each diagnostic,
|
||||
# even in an image that had already validated all of them at build.
|
||||
# Keep an explicit full recheck for diagnosing a suspect installation.
|
||||
binary_record["no_cache"] = no_cache
|
||||
signature = json.dumps(binary_record, sort_keys=True, default=str)
|
||||
declared_binary_specs.setdefault(signature, binary_record)
|
||||
|
||||
@@ -330,15 +340,9 @@ def version(
|
||||
for dependency in plugins.select([plugin_name]).values()
|
||||
if not config.get(dependency.enabled_key, True)
|
||||
)
|
||||
binary_records = get_required_binary_requests(
|
||||
plugin,
|
||||
plugin.config.required_binaries,
|
||||
overrides=runtime_config,
|
||||
derived_overrides=derived_config,
|
||||
run_output_dir=CONSTANTS.DATA_DIR,
|
||||
)
|
||||
binary_records = binary_records_by_plugin[plugin_name]
|
||||
for binary_record in binary_records:
|
||||
binary_record["no_cache"] = True
|
||||
binary_record["no_cache"] = no_cache
|
||||
actual_name = str(binary_record["name"])
|
||||
logical_name = (
|
||||
Path(actual_name).expanduser().name
|
||||
@@ -485,6 +489,11 @@ def version(
|
||||
is_flag=True,
|
||||
help="Only print ArchiveBox version number and nothing else. (equivalent to archivebox --version)",
|
||||
)
|
||||
@click.option(
|
||||
"--no-cache",
|
||||
is_flag=True,
|
||||
help="Reprobe binary versions and hashes instead of reusing validated provider caches.",
|
||||
)
|
||||
@click.option(
|
||||
"--binaries",
|
||||
"-b",
|
||||
|
||||
@@ -414,6 +414,36 @@ class ServerConfig(BaseConfigSet):
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def _default_sqlite_journal_mode(database_name: str) -> str:
|
||||
if not IN_DOCKER:
|
||||
return "WAL"
|
||||
|
||||
# Native Linux bind mounts share the kernel's locks and mmap; Docker alone
|
||||
# does not make WAL unsafe. DELETE there lets any slow reader block tiny
|
||||
# writes. Desktop VM shares (virtiofs/9p/FUSE), network filesystems and
|
||||
# unknown mounts keep rollback journaling: their host-side readers may not
|
||||
# share SQLite's -shm locking domain. Inspect the DB mount, not archive/.
|
||||
database_path = Path(database_name).expanduser().resolve()
|
||||
try:
|
||||
mounts = Path("/proc/self/mountinfo").read_text().splitlines()
|
||||
except OSError:
|
||||
return "DELETE"
|
||||
filesystem = ""
|
||||
mount_depth = -1
|
||||
for line in mounts:
|
||||
fields, separator, details = line.partition(" - ")
|
||||
if not separator:
|
||||
continue
|
||||
mount_path = fields.split()[4]
|
||||
mount_path = re.sub(r"\\([0-7]{3})", lambda match: chr(int(match[1], 8)), mount_path)
|
||||
mount = Path(mount_path)
|
||||
if database_path.is_relative_to(mount) and len(mount.parts) > mount_depth:
|
||||
filesystem = details.split()[0]
|
||||
mount_depth = len(mount.parts)
|
||||
return "WAL" if filesystem in {"ext2", "ext3", "ext4", "xfs", "btrfs", "zfs", "overlay", "tmpfs", "f2fs"} else "DELETE"
|
||||
|
||||
|
||||
class DatabaseConfig(BaseConfigSet):
|
||||
toml_section_header: str = "DATABASE_CONFIG"
|
||||
_scope: str = PrivateAttr(default=_SCOPE_SERVER)
|
||||
@@ -429,11 +459,7 @@ class DatabaseConfig(BaseConfigSet):
|
||||
DATABASE_USER: str = Field(default="archivebox", alias="ARCHIVEBOX_DATABASE_USER")
|
||||
DATABASE_PASSWORD: str = Field(default="", alias="ARCHIVEBOX_DATABASE_PASSWORD")
|
||||
SQLITE_JOURNAL_MODE: str = Field(
|
||||
# Docker collections commonly live on a host bind mount. WAL's -shm
|
||||
# locking is only safe when every SQLite process is on the same host;
|
||||
# Docker Desktop/OrbStack place the container and host in different
|
||||
# locking domains and a host-side reader can corrupt the live DB.
|
||||
default="DELETE" if IN_DOCKER else "WAL",
|
||||
default_factory=lambda data: _default_sqlite_journal_mode(str(data["DATABASE_NAME"])),
|
||||
alias="ARCHIVEBOX_SQLITE_JOURNAL_MODE",
|
||||
pattern=r"(?i)^(DELETE|TRUNCATE|PERSIST|MEMORY|WAL|OFF)$",
|
||||
)
|
||||
@@ -447,11 +473,16 @@ class DatabaseConfig(BaseConfigSet):
|
||||
SQLITE_LOCK_RETRY_INTERVAL: float = Field(default=5.0, alias="ARCHIVEBOX_SQLITE_LOCK_RETRY_INTERVAL", gt=0)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def reject_docker_sqlite_wal(self):
|
||||
if IN_DOCKER and self.DATABASE_ENGINE.lower() == "sqlite" and self.SQLITE_JOURNAL_MODE.upper() == "WAL":
|
||||
def reject_shared_filesystem_sqlite_wal(self):
|
||||
if (
|
||||
IN_DOCKER
|
||||
and self.DATABASE_ENGINE.lower() == "sqlite"
|
||||
and self.SQLITE_JOURNAL_MODE.upper() == "WAL"
|
||||
and _default_sqlite_journal_mode(self.DATABASE_NAME) != "WAL"
|
||||
):
|
||||
raise ValueError(
|
||||
"SQLITE_JOURNAL_MODE=WAL is unsafe for Docker collections because host bind mounts cross SQLite "
|
||||
"locking domains; use DELETE (the Docker default) or PostgreSQL",
|
||||
"SQLITE_JOURNAL_MODE=WAL requires a local filesystem with shared SQLite locking; "
|
||||
"keep the database on a local Docker volume, use DELETE for host/VM shared mounts, or use PostgreSQL",
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
@@ -52,7 +52,6 @@ from archivebox.core.models import Tag, Snapshot, ArchiveResult
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.core.admin_archiveresults import render_archiveresults_list
|
||||
from archivebox.progressmonitor.views import progress_endpoint
|
||||
from archivebox.workers.models import RETRY_AT_MAX
|
||||
from archivebox.core.permissions import (
|
||||
PERMISSIONS_CHOICES,
|
||||
PERMISSIONS_META,
|
||||
@@ -1544,7 +1543,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
|
||||
)
|
||||
def delete_snapshots(self, request, queryset):
|
||||
"""Keep the rows until the runner removes and verifies their files."""
|
||||
total = queryset.update(status=Snapshot.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
|
||||
total = queryset.request_delete()
|
||||
messages.success(
|
||||
request,
|
||||
f"Queued {total} snapshots for deletion. They remain visible as Deleting until ArchiveBox verifies their files are removed.",
|
||||
|
||||
@@ -31,7 +31,7 @@ django_application = ArchiveBoxASGIHandler()
|
||||
|
||||
async def application(scope, receive, send):
|
||||
if scope["type"] == "websocket":
|
||||
from archivebox.opencode.views import websocket_view
|
||||
from abx_plugins.plugins.opencode.archivebox.views import websocket_view
|
||||
|
||||
return await websocket_view(scope, receive, send)
|
||||
return await django_application(scope, receive, send)
|
||||
@@ -177,6 +177,12 @@ class SnapshotTag(models.Model):
|
||||
class SnapshotQuerySet(PagedQuerySet):
|
||||
"""Custom QuerySet for Snapshot model with export methods that persist through .filter() etc."""
|
||||
|
||||
def request_delete(self) -> int:
|
||||
# Admin and API deletion share this durable command. Keep the row until
|
||||
# the runner stops its writers and verifies removal from remote storage;
|
||||
# RETRY_AT_MAX prevents capture scheduling while cleanup is pending.
|
||||
return self.update(status=self.model.DELETING_STATE, retry_at=RETRY_AT_MAX, modified_at=timezone.now())
|
||||
|
||||
def delete(self):
|
||||
# Finish the SELECT before filesystem work, so no read cursor or SQLite
|
||||
# transaction is held while deleting remote archive payloads.
|
||||
@@ -598,7 +604,7 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
using = using or self._state.db or "default"
|
||||
if connections[using].in_atomic_block:
|
||||
raise RuntimeError("Snapshot filesystem deletion must run outside a database transaction")
|
||||
type(self).objects.using(using).filter(pk=self.pk).update(status=self.DELETING_STATE, retry_at=RETRY_AT_MAX)
|
||||
type(self).objects.using(using).filter(pk=self.pk).request_delete()
|
||||
self.status = self.DELETING_STATE
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
@@ -875,8 +881,11 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
|
||||
@classmethod
|
||||
def missing_delete_at_candidates(cls):
|
||||
return cls.objects.filter(delete_at__isnull=True).filter(
|
||||
Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"),
|
||||
return (
|
||||
cls.objects.filter(delete_at__isnull=True)
|
||||
.filter(Q(config__has_key="DELETE_AFTER") | Q(crawl__config__has_key="DELETE_AFTER"))
|
||||
.select_related("crawl")
|
||||
.only("id", "created_at", "delete_at", "config", "crawl__config")
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -1194,7 +1203,9 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
def hydrate_archiveresult_output_metadata(self, snapshot_dir: Path | None = None) -> int:
|
||||
"""Populate missing ArchiveResult file metadata from existing outputs."""
|
||||
hydrated = 0
|
||||
for result in self.archiveresult_set.filter(output_files={}).iterator():
|
||||
# Files may live on slow remote storage: finish the bounded read before
|
||||
# inspecting outputs or writing their metadata back to the database.
|
||||
for result in self.archiveresult_set.filter(output_files={}).order_by("pk").paged_iterator():
|
||||
hydrated += int(result.update_output_metadata_from_filesystem(snapshot_dir=snapshot_dir))
|
||||
return hydrated
|
||||
|
||||
@@ -3939,6 +3950,8 @@ class Snapshot(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelW
|
||||
|
||||
|
||||
class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
objects = PagedQuerySet.as_manager()
|
||||
|
||||
class StatusChoices(models.TextChoices):
|
||||
QUEUED = "queued", "Queued"
|
||||
STARTED = "started", "Started"
|
||||
@@ -4160,8 +4173,11 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
|
||||
@classmethod
|
||||
def missing_delete_at_candidates(cls):
|
||||
return cls.objects.filter(delete_at__isnull=True).filter(
|
||||
Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"),
|
||||
return (
|
||||
cls.objects.filter(delete_at__isnull=True)
|
||||
.filter(Q(snapshot__config__has_key="DELETE_AFTER") | Q(snapshot__crawl__config__has_key="DELETE_AFTER"))
|
||||
.select_related("snapshot__crawl")
|
||||
.only("id", "created_at", "delete_at", "snapshot__config", "snapshot__crawl__config")
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -26,15 +26,14 @@ from archivebox.core.views import (
|
||||
)
|
||||
from archivebox.progressmonitor.views import live_progress_view
|
||||
from archivebox.search.views import public_snapshot_search_stream_view
|
||||
from archivebox.opencode.views import opencode_proxy_view
|
||||
from abx_plugins.plugins.opencode.archivebox.urls import urlpatterns as agent_urlpatterns
|
||||
|
||||
urlpatterns = [
|
||||
re_path(r"^static/(?P<path>.*)$", serve_static),
|
||||
path("robots.txt", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "robots.txt"}),
|
||||
path("favicon.ico", static.serve, {"document_root": CONSTANTS.STATIC_DIR, "path": "favicon.ico"}),
|
||||
path("docs/", RedirectView.as_view(url="https://github.com/ArchiveBox/ArchiveBox/wiki"), name="Docs"),
|
||||
re_path(r"^admin/agent/?(?=$|opencode)", include("archivebox.opencode.urls")),
|
||||
re_path(r"^(?P<path>assets/.*)$", opencode_proxy_view, name="opencode-assets"),
|
||||
*agent_urlpatterns,
|
||||
path("public/search-stream/", public_snapshot_search_stream_view, name="public-search-stream"),
|
||||
path("public/", PublicIndexView.as_view(), name="public-index"),
|
||||
path("public.html", RedirectView.as_view(url="/public/"), name="public-index-html"),
|
||||
|
||||
+32
-27
@@ -122,7 +122,7 @@ def _find_snapshot_by_ref(snapshot_ref: str) -> Snapshot | None:
|
||||
if len(lookup) == 12 and "-" not in lookup:
|
||||
# Resolve suffixes using the covering primary-key index before fetching
|
||||
# wide snapshot rows and joining their crawl/user data.
|
||||
matches = list(Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True))
|
||||
matches = Snapshot.objects.filter(id__endswith=lookup).order_by().values_list("id", flat=True)
|
||||
return snapshots.filter(pk__in=matches).order_by("-created_at", "-downloaded_at").first()
|
||||
|
||||
try:
|
||||
@@ -506,7 +506,7 @@ class SnapshotView(View):
|
||||
def _resolve_snapshots_for_slug(slug: str):
|
||||
# full URLs go straight to the url-only path (fast, indexed)
|
||||
if "://" in slug:
|
||||
return SnapshotView.find_snapshots_for_url(slug)
|
||||
return SnapshotView.find_snapshots_for_url(slug, allow_fallback=False)
|
||||
# short uuid-shaped slugs (>=8 hex chars after stripping non-hex) try id matching first
|
||||
id_qs = SnapshotView.find_snapshots_for_id(slug)
|
||||
if id_qs.exists():
|
||||
@@ -517,6 +517,12 @@ class SnapshotView(View):
|
||||
try:
|
||||
if "://" in path:
|
||||
snapshot = snapshots.order_by("-bookmarked_at").first()
|
||||
if snapshot is None:
|
||||
# Fetch the exact match once before trying legacy URL
|
||||
# fallbacks; an exists() probe followed by the same lookup
|
||||
# needlessly doubled reads on every normal redirect.
|
||||
snapshots = direct_snapshots_queryset(request, self.find_snapshots_for_url(path)).select_related("crawl__created_by")
|
||||
snapshot = snapshots.order_by("-bookmarked_at").first()
|
||||
if snapshot is None:
|
||||
raise Snapshot.DoesNotExist
|
||||
else:
|
||||
@@ -1228,7 +1234,6 @@ class PublicIndexView(ListView):
|
||||
model = Snapshot
|
||||
ordering: ClassVar[list[str]] = ["-bookmarked_at", "-created_at"]
|
||||
paginator_class = AcceleratedPaginator
|
||||
public_page_scan_chunk_size = 50
|
||||
|
||||
def get_paginate_by(self, queryset):
|
||||
runtime_config = self.__dict__.get("runtime_config")
|
||||
@@ -1256,31 +1261,31 @@ class PublicIndexView(ListView):
|
||||
)
|
||||
|
||||
def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
|
||||
# One extra row determines Next without a full collection count. Cached
|
||||
# totals can be stale while the runner adds/removes snapshots, so they
|
||||
# must never decide whether the requested page exists.
|
||||
target_count = page_number * page_size + 1
|
||||
public_snapshots: list[Snapshot] = []
|
||||
scanned = 0
|
||||
chunk_size = max(self.public_page_scan_chunk_size, page_size + 1)
|
||||
# Read in snapshot_public_order_idx order and filter visibility in small
|
||||
# batches. Combining the permissions predicate with ORDER BY can make
|
||||
# SQLite select the permissions index and sort the whole public archive
|
||||
# before LIMIT. This favors the common early pages without a new index;
|
||||
# deep pages or mostly private collections can still require more reads.
|
||||
ordered_snapshots = (
|
||||
Snapshot.objects.select_related("crawl__created_by").order_by(*self.ordering).only(*self._base_public_snapshot_fields())
|
||||
)
|
||||
|
||||
while len(public_snapshots) < target_count:
|
||||
chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
|
||||
if not chunk:
|
||||
break
|
||||
scanned += len(chunk)
|
||||
public_snapshots.extend(snapshot for snapshot in chunk if snapshot.permissions == PERMISSIONS_PUBLIC)
|
||||
|
||||
# Walk the existing chronological index in bounded keyset pages,
|
||||
# reading only identity/visibility until reaching the requested page.
|
||||
# Repeated OFFSET batches used to revisit every earlier row AND join
|
||||
# owners each time, making deep pages quadratic. Filtering visibility
|
||||
# in SQL instead made SQLite sort the entire permissions index first.
|
||||
# Each helper page closes its cursor before yielding; only the final
|
||||
# displayed IDs load wide rows/owners. No new index or SQLite hint.
|
||||
start = (page_number - 1) * page_size
|
||||
return public_snapshots[start:target_count]
|
||||
visible_count = 0
|
||||
page_ids = []
|
||||
candidates = Snapshot.objects.only("id", "permissions").order_by(*self.ordering, "id")
|
||||
for snapshot in candidates.paged_iterator(chunk_size=max(100, page_size + 1)):
|
||||
if snapshot.permissions != PERMISSIONS_PUBLIC:
|
||||
continue
|
||||
if visible_count >= start:
|
||||
page_ids.append(snapshot.pk)
|
||||
if len(page_ids) > page_size:
|
||||
break
|
||||
visible_count += 1
|
||||
return list(
|
||||
Snapshot.objects.filter(pk__in=page_ids, permissions=PERMISSIONS_PUBLIC)
|
||||
.select_related("crawl__created_by")
|
||||
.order_by(*self.ordering, "id")
|
||||
.only(*self._base_public_snapshot_fields()),
|
||||
)
|
||||
|
||||
def paginate_queryset(self, queryset, page_size):
|
||||
if self.request.GET.get("q", default="").strip():
|
||||
|
||||
@@ -947,7 +947,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
|
||||
updated = 0
|
||||
batch = []
|
||||
crawls_to_update = []
|
||||
for crawl in queryset.only("id", "config", "permissions").iterator(chunk_size=500):
|
||||
for crawl in queryset.only("id", "config", "permissions").order_by("pk").paged_iterator(chunk_size=500):
|
||||
old_permissions = crawl.permissions
|
||||
config = dict(crawl.config or {})
|
||||
config["PERMISSIONS"] = permissions
|
||||
|
||||
@@ -23,6 +23,7 @@ from archivebox.config.common import rprint as print
|
||||
from archivebox.core.permissions import PERMISSIONS_VALUES, normalize_permissions
|
||||
|
||||
from archivebox.base_models.models import (
|
||||
PagedQuerySet,
|
||||
ModelWithUUID,
|
||||
ModelWithDeleteAfter,
|
||||
ModelWithOutputDir,
|
||||
@@ -148,6 +149,8 @@ class CrawlSchedule(ModelWithUUID, ModelWithNotes):
|
||||
|
||||
|
||||
class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWithQueue):
|
||||
objects = PagedQuerySet.as_manager()
|
||||
|
||||
id = CompactUUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
|
||||
created_at = models.DateTimeField(default=timezone.now, db_index=True)
|
||||
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
|
||||
@@ -229,7 +232,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
if paused and save and self.pk:
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).iterator():
|
||||
for snapshot in self.snapshot_set.exclude(status__in=Snapshot.FINAL_STATES).order_by("pk").paged_iterator():
|
||||
snapshot.pause()
|
||||
return paused
|
||||
|
||||
@@ -301,7 +304,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
|
||||
@classmethod
|
||||
def missing_delete_at_candidates(cls):
|
||||
return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER")
|
||||
return cls.objects.filter(delete_at__isnull=True, config__has_key="DELETE_AFTER").only("id", "created_at", "delete_at", "config")
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
update_fields = kwargs.get("update_fields")
|
||||
@@ -368,7 +371,7 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
batch = []
|
||||
updated = 0
|
||||
queryset = self.snapshot_set.filter(Q(permissions=old_permissions) | Q(permissions__isnull=True)).only("id", "config")
|
||||
for snapshot in queryset.iterator(chunk_size=500):
|
||||
for snapshot in queryset.order_by("pk").paged_iterator(chunk_size=500):
|
||||
config = dict(snapshot.config or {})
|
||||
config["PERMISSIONS"] = normalized_new_permissions
|
||||
snapshot.config = config
|
||||
@@ -427,7 +430,11 @@ class Crawl(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithConfig, ModelWith
|
||||
tags_by_name = {tag.name: tag for tag in Tag.objects.filter(name__in=added_names)}
|
||||
|
||||
tag_ids = [tag.pk for tag_name in added_names if (tag := tags_by_name.get(tag_name))]
|
||||
snapshot_ids = Snapshot.objects.filter(crawl=self).values_list("id", flat=True).iterator(chunk_size=5000)
|
||||
# Fetch each page before writing M2M rows. iterator(chunk_size=...)
|
||||
# would keep one read cursor open across every bulk write.
|
||||
snapshot_ids = (
|
||||
snapshot.pk for snapshot in Snapshot.objects.filter(crawl=self).only("id").order_by("pk").paged_iterator(chunk_size=500)
|
||||
)
|
||||
while True:
|
||||
batch_snapshot_ids = list(islice(snapshot_ids, 5000))
|
||||
if not batch_snapshot_ids:
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
def repair_binary_field_indexes(apps, schema_editor):
|
||||
Binary = apps.get_model("machine", "Binary")
|
||||
connection = schema_editor.connection
|
||||
with connection.cursor() as cursor:
|
||||
constraints = connection.introspection.get_constraints(cursor, Binary._meta.db_table)
|
||||
indexed_columns = {
|
||||
tuple(constraint["columns"])
|
||||
for constraint in constraints.values()
|
||||
if constraint.get("index") or constraint.get("unique") or constraint.get("primary_key")
|
||||
}
|
||||
# 0005's legacy raw-SQL table creation omitted field indexes even though
|
||||
# migration state declares them. Editing 0005 alone cannot repair already
|
||||
# upgraded collections. Restore only missing single-field indexes, checking
|
||||
# columns rather than generated names so fresh SQLite/PostgreSQL installs
|
||||
# retain their existing indexes without duplicates or a schema-state change.
|
||||
for field in Binary._meta.local_fields:
|
||||
if field.db_index and not field.unique and (field.column,) not in indexed_columns:
|
||||
schema_editor.add_index(Binary, models.Index(fields=[field.name], name=f"mach_binary_{field.column}_idx"))
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("machine", "0022_networkinterface_identity_without_mac")]
|
||||
|
||||
operations = [migrations.RunPython(repair_binary_field_indexes, migrations.RunPython.noop)]
|
||||
@@ -48,7 +48,6 @@ _CURRENT_PROCESS: Process | None = None
|
||||
MACHINE_RECHECK_INTERVAL = 7 * 24 * 60 * 60
|
||||
NETWORK_INTERFACE_RECHECK_INTERVAL = 1 * 60 * 60
|
||||
BINARY_RECHECK_INTERVAL = 1 * 30 * 60
|
||||
PROCESS_RECHECK_INTERVAL = 60 # Re-validate every 60 seconds
|
||||
PID_REUSE_WINDOW = timedelta(hours=24) # Max age for considering a PID match valid
|
||||
PROCESS_TIMEOUT_GRACE = timedelta(seconds=30) # Extra margin before force-cleaning timed-out RUNNING rows
|
||||
START_TIME_TOLERANCE = 5.0 # Seconds tolerance for start time matching
|
||||
@@ -473,7 +472,7 @@ class NetworkInterface(ModelWithHealthStats):
|
||||
return _CURRENT_INTERFACE
|
||||
|
||||
|
||||
class BinaryManager(models.Manager):
|
||||
class BinaryManager(models.Manager.from_queryset(PagedQuerySet)):
|
||||
def get_from_db_or_cache(self, name: str, abspath: str = "", version: str = "", sha256: str = "", binprovider: str = "env") -> Binary:
|
||||
"""Get or create an Binary record from the database or cache."""
|
||||
cached = _CURRENT_BINARIES.get(name)
|
||||
@@ -1209,8 +1208,11 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
|
||||
@classmethod
|
||||
def missing_delete_at_candidates(cls):
|
||||
return cls.objects.filter(delete_at__isnull=True).filter(
|
||||
Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"),
|
||||
return (
|
||||
cls.objects.filter(delete_at__isnull=True)
|
||||
.filter(Q(env__has_key="DELETE_AFTER") | Q(machine__config__has_key="DELETE_AFTER"))
|
||||
.select_related("machine")
|
||||
.only("id", "created_at", "delete_at", "env", "machine__config")
|
||||
)
|
||||
|
||||
# Properties that delegate to related objects
|
||||
@@ -1410,6 +1412,13 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
# Process.current() and hierarchy methods
|
||||
# =========================================================================
|
||||
|
||||
@classmethod
|
||||
def current_readonly(cls) -> Process | None:
|
||||
"""Return this process's registered identity without registration or probes."""
|
||||
if _CURRENT_PROCESS is not None and _CURRENT_PROCESS.pid == os.getpid():
|
||||
return _CURRENT_PROCESS
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def current(cls) -> Process:
|
||||
"""
|
||||
@@ -1427,40 +1436,19 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
|
||||
current_pid = os.getpid()
|
||||
|
||||
# Fast path used by model save diagnostics and hot runner loops. A
|
||||
# cached Process object is valid when the immutable identity we wrote
|
||||
# at creation time still describes this Python process. PID reuse cannot
|
||||
# happen while this process is alive, so pid + present started_at/cmd is
|
||||
# enough here; the slower psutil validation below remains the fallback
|
||||
# for missing/stale cache.
|
||||
if (
|
||||
_CURRENT_PROCESS
|
||||
and _CURRENT_PROCESS.pid == current_pid
|
||||
and _CURRENT_PROCESS.status == cls.StatusChoices.RUNNING
|
||||
and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
|
||||
and _CURRENT_PROCESS.started_at is not None
|
||||
and bool(_CURRENT_PROCESS.cmd)
|
||||
):
|
||||
return _CURRENT_PROCESS
|
||||
# This is process identity, not a health check. A live Python process
|
||||
# cannot reuse its own PID, and a fork fails current_readonly's PID
|
||||
# check. Expiring this cache by modified_at made every call after 60s
|
||||
# query the database and touch logs until something else saved the row.
|
||||
# Explicit lifecycle/recovery code owns liveness and interface refresh.
|
||||
cached = cls.current_readonly()
|
||||
if cached is not None:
|
||||
return cached
|
||||
_CURRENT_PROCESS = None
|
||||
|
||||
machine = Machine.current()
|
||||
iface = NetworkInterface.current()
|
||||
|
||||
# Check cache validity
|
||||
if _CURRENT_PROCESS:
|
||||
# Verify: same PID, same machine, cache not expired
|
||||
if (
|
||||
_CURRENT_PROCESS.pid == current_pid
|
||||
and _CURRENT_PROCESS.machine_id == machine.id
|
||||
and timezone.now() < _CURRENT_PROCESS.modified_at + timedelta(seconds=PROCESS_RECHECK_INTERVAL)
|
||||
):
|
||||
if _CURRENT_PROCESS.iface_id != iface.id:
|
||||
_CURRENT_PROCESS.iface = iface
|
||||
_CURRENT_PROCESS.save(update_fields=["iface", "modified_at"])
|
||||
_CURRENT_PROCESS.ensure_log_files()
|
||||
return _CURRENT_PROCESS
|
||||
_CURRENT_PROCESS = None
|
||||
|
||||
# Get actual process start time from OS for validation
|
||||
os_start_time = None
|
||||
if PSUTIL_AVAILABLE:
|
||||
@@ -2581,9 +2569,13 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
"""
|
||||
cleaned = 0
|
||||
|
||||
# Select active IDs through the existing status index first. Combining
|
||||
# type/status directly let SQLite prefer the historical type index and
|
||||
# scan 350k processes to return zero workers (blocking writes for 12s).
|
||||
active_ids = cls.objects.filter(status=cls.StatusChoices.RUNNING).order_by().values("pk")
|
||||
running_children = cls.objects.filter(
|
||||
pk__in=active_ids,
|
||||
process_type__in=[cls.TypeChoices.WORKER, cls.TypeChoices.HOOK],
|
||||
status=cls.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
# Close each bounded read before inspecting processes or writing their state.
|
||||
|
||||
+23
-74
@@ -240,89 +240,38 @@ def drop_models_on_postgres(apps, schema_editor, app_label: str, model_names: li
|
||||
schema_editor.delete_model(model)
|
||||
|
||||
|
||||
def run_db_analyze_batch(
|
||||
remaining: list[str] | None,
|
||||
*,
|
||||
max_seconds_per_table: float = 120.0,
|
||||
) -> list[str]:
|
||||
"""Advance one step of a batched SQLite ``ANALYZE`` sweep.
|
||||
def run_db_analyze_batch(remaining: list[str] | None) -> list[str]:
|
||||
"""Refresh approximate SQLite planner stats for one table per idle tick.
|
||||
|
||||
Without periodic ANALYZE the optimizer's table stats go stale as
|
||||
snapshot/archiveresult tables grow, causing it to start large joins from
|
||||
``auth_user`` instead of using the indexed url column and blowing snapshot
|
||||
detail page render time from ~50ms to ~500ms+.
|
||||
|
||||
The whole sweep is spread across many calls instead of running as one
|
||||
blocking ``ANALYZE``: pass ``None`` to start a fresh sweep (this call
|
||||
enumerates user tables and runs ``ANALYZE`` on the first one); pass the
|
||||
returned list to advance one more table on each subsequent call. An
|
||||
empty return value means the sweep is complete (or has been aborted) and
|
||||
the next caller should pass ``None`` again. Caller is responsible for
|
||||
throttling new sweeps (orchestrator starts at most one per 24hr while
|
||||
idle) and enforcing a hard upper bound on total sweep wall time.
|
||||
|
||||
Safety guarantees:
|
||||
|
||||
- **Never raises**: every database call is wrapped; on any failure the
|
||||
function returns ``[]`` (abandoning the rest of the sweep) so the
|
||||
orchestrator never crashes on maintenance errors.
|
||||
- **Bounded per-call wall time**: a SQLite progress handler aborts the
|
||||
current ``ANALYZE`` statement once ``max_seconds_per_table`` is
|
||||
exceeded, so a single pathological table cannot wedge the call.
|
||||
- **Never leaves the db locked**: each ``ANALYZE`` runs as a single
|
||||
statement transaction that auto-commits (or rolls back on
|
||||
abort/error). The cursor and progress handler are always cleaned up
|
||||
in ``finally`` blocks even if Python raises mid-call.
|
||||
- Silent no-op on non-SQLite backends.
|
||||
|
||||
WAL journal mode (set in Django settings) keeps readers fully unblocked
|
||||
throughout; the writer lock is only held for the brief ``sqlite_stat*``
|
||||
flush after each table completes.
|
||||
A full ANALYZE scans every index while holding a write transaction; moving
|
||||
it into the runner did not prevent it from blocking submissions. Sampling
|
||||
at most about 1000 entries per index keeps maintenance independent of
|
||||
collection size. Approximate stats are sufficient for choosing indexes.
|
||||
PostgreSQL owns its statistics through autovacuum/autoanalyze.
|
||||
"""
|
||||
from django.db import connection
|
||||
|
||||
if connection.vendor != "sqlite":
|
||||
return []
|
||||
|
||||
if remaining is None:
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
|
||||
)
|
||||
remaining = [row[0] for row in cursor.fetchall()]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
if not remaining:
|
||||
return []
|
||||
|
||||
next_table, *rest = remaining
|
||||
raw_conn = connection.connection
|
||||
progress_handler_set = False
|
||||
if raw_conn is not None and max_seconds_per_table > 0:
|
||||
deadline = time.monotonic() + max_seconds_per_table
|
||||
try:
|
||||
raw_conn.set_progress_handler(lambda: 1 if time.monotonic() > deadline else 0, 10000)
|
||||
progress_handler_set = True
|
||||
except Exception:
|
||||
progress_handler_set = False
|
||||
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(f'ANALYZE "{next_table}"')
|
||||
except Exception:
|
||||
# Aborted by progress handler, locked db, or any other failure — skip
|
||||
# this table and continue the sweep. ANALYZE is idempotent so we can
|
||||
# retry on the next 24hr sweep.
|
||||
pass
|
||||
finally:
|
||||
if progress_handler_set and raw_conn is not None:
|
||||
if remaining is None:
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")
|
||||
remaining = [row[0] for row in cursor.fetchall()]
|
||||
if not remaining:
|
||||
return []
|
||||
next_table, *rest = remaining
|
||||
cursor.execute("PRAGMA analysis_limit")
|
||||
previous_limit = int(cursor.fetchone()[0])
|
||||
try:
|
||||
raw_conn.set_progress_handler(None, 0)
|
||||
except Exception:
|
||||
pass
|
||||
return rest
|
||||
cursor.execute("PRAGMA analysis_limit = 1000")
|
||||
cursor.execute(f"ANALYZE {connection.ops.quote_name(next_table)}")
|
||||
finally:
|
||||
cursor.execute(f"PRAGMA analysis_limit = {previous_limit}")
|
||||
return rest
|
||||
except Exception:
|
||||
# Optional planner maintenance must not prevent queued captures.
|
||||
return []
|
||||
|
||||
|
||||
def compact_command(cmdline: list[str] | None, fallback: str = "") -> str:
|
||||
|
||||
@@ -142,3 +142,27 @@ def write_record(record: dict[str, Any], stream: TextIO | None = None) -> None:
|
||||
active_stream: TextIO = sys.stdout if stream is None else stream
|
||||
active_stream.write(json.dumps(record) + "\n")
|
||||
active_stream.flush()
|
||||
|
||||
|
||||
def write_records(records: Iterable[dict[str, Any]], stream: TextIO | None = None) -> int:
|
||||
"""Stream bulk JSONL output with bounded buffering, returning its row count."""
|
||||
active_stream: TextIO = sys.stdout if stream is None else stream
|
||||
lines: list[str] = []
|
||||
pending_size = 0
|
||||
count = 0
|
||||
for record in records:
|
||||
line = json.dumps(record) + "\n"
|
||||
lines.append(line)
|
||||
pending_size += len(line)
|
||||
count += 1
|
||||
# CLI stdout can be unbuffered. A syscall per snapshot makes large
|
||||
# exports IO-bound; small batches keep memory and pipe latency bounded.
|
||||
if pending_size >= 64 * 1024:
|
||||
active_stream.write("".join(lines))
|
||||
active_stream.flush()
|
||||
lines.clear()
|
||||
pending_size = 0
|
||||
if lines:
|
||||
active_stream.write("".join(lines))
|
||||
active_stream.flush()
|
||||
return count
|
||||
@@ -32,6 +32,7 @@ from archivebox.misc.logging_util import printable_filesize
|
||||
from archivebox.plugins.discovery import render_plugin_full_response
|
||||
|
||||
_HASHES_CACHE: dict[Path, tuple[float, dict[str, str]]] = {}
|
||||
DIRECTORY_PREVIEW_MAX_BYTES = 100 * 1024
|
||||
FAVICON_CACHE_CONTROL = "public, max-age=31536000, s-maxage=31536000, immutable"
|
||||
IMG_SRC_ATTR_RE = re.compile(r'(<img\b[^>]*?\s(?:src|data-src)=["\'])([^"\']+)(["\'])', re.IGNORECASE)
|
||||
TRANSFORMED_HTML_PREVIEW_STYLE = """<style id="archivebox-static-html-preview-style">
|
||||
@@ -69,6 +70,192 @@ a > img:not([width]):not([height]) {
|
||||
</style>"""
|
||||
|
||||
|
||||
def add_replay_image_fallback(response: HttpResponse) -> HttpResponse:
|
||||
"""Add a best-effort responsive-image fallback to the locally served replay worker.
|
||||
|
||||
WHY: a recorder saves the image Chrome selected, not every srcset candidate.
|
||||
Replaying at another viewport/DPR can select an uncaptured URL (the motivating
|
||||
case was Drive's missing 2x logo with its 1x logo already in the same WACZ).
|
||||
WACZ requests are answered inside Webrecorder's service worker, so a Django
|
||||
404 handler cannot repair them. Append this adapter when serving that worker;
|
||||
do not change captures, request more live assets, or resize the recorded page.
|
||||
|
||||
Only an image GET that already returned 404 is eligible. The worker reads the
|
||||
referring HTML from the SAME archive and tries src/srcset candidates declared
|
||||
on that img or its enclosing picture. It never guesses filename substitutions
|
||||
or borrows an unrelated nearby image. The original response wins on absent
|
||||
metadata, unsupported worker APIs, parse errors, timeout, or failed candidates.
|
||||
This deliberately does not promise to fix JS-only srcsets, CSS backgrounds,
|
||||
corrupt images returning 200, or pages whose archived HTML is too large.
|
||||
|
||||
Keep this self-contained in the static-file server: it is a replay repair,
|
||||
independent of the capturing plugin/site. Feature detection makes upstream
|
||||
worker changes disable the repair rather than break ordinary replay. The
|
||||
archive's HTML/ZIP bytes and successful HTTP responses remain untouched.
|
||||
"""
|
||||
if (
|
||||
response.status_code != 200
|
||||
or response.streaming
|
||||
or not response.has_header("Service-Worker-Allowed")
|
||||
or "javascript" not in response.get("Content-Type", "")
|
||||
):
|
||||
return response
|
||||
script = rb"""
|
||||
;(() => {
|
||||
try {
|
||||
const replay = self.sw;
|
||||
if (!replay || typeof replay.handleFetch !== "function" ||
|
||||
typeof replay.getResponseFor !== "function" || !replay.collections ||
|
||||
replay.proxyOriginMode || replay.topFramePassthrough || replay.__abxImageFallback) return;
|
||||
const original = replay.handleFetch;
|
||||
const documents = new Map();
|
||||
const decode = value => value.replace(/&(?:amp|quot|apos|lt|gt|#\d+|#x[\da-f]+);/gi, entity => {
|
||||
const named = {"&":"&", """:'"', "'":"'", "<":"<", ">":">"};
|
||||
return named[entity.toLowerCase()] || String.fromCodePoint(
|
||||
entity[2].toLowerCase() === "x" ? parseInt(entity.slice(3), 16) : parseInt(entity.slice(2), 10));
|
||||
});
|
||||
const attributes = tag => {
|
||||
const attrs = {};
|
||||
for (const m of tag.matchAll(/([^\s=/>]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/g)) {
|
||||
attrs[m[1].toLowerCase()] = decode(m[2] ?? m[3] ?? m[4]);
|
||||
}
|
||||
return attrs;
|
||||
};
|
||||
const sourceSet = value => {
|
||||
// URLs may contain commas (e.g. image-CDN transforms). Like HTML's
|
||||
// srcset tokenizer, consume a URL through whitespace, then descriptors.
|
||||
const urls = [];
|
||||
let rest = value || "";
|
||||
while (rest && urls.length < 64) {
|
||||
rest = rest.replace(/^[\s,]+/, "");
|
||||
const token = rest.match(/^\S+/)?.[0];
|
||||
if (!token) break;
|
||||
urls.push(token.replace(/,+$/, ""));
|
||||
rest = rest.slice(token.length);
|
||||
if (!token.endsWith(",")) rest = rest.replace(/^[^,]*(?:,|$)/, "");
|
||||
}
|
||||
return urls;
|
||||
};
|
||||
const splitReplay = (url, prefix) => {
|
||||
if (!url.startsWith(prefix)) return null;
|
||||
return url.slice(prefix.length).match(/^((?::[^/]+\/)?\d{0,14})(?:[a-z]+_)?\/((?:https?:)?\/\/.+)$/);
|
||||
};
|
||||
replay.handleFetch = async function(event) {
|
||||
// Never catch/retry the original operation: preserve its normal behavior.
|
||||
const response = await original.call(this, event);
|
||||
const request = event.request;
|
||||
if (response.status !== 404 || request.method !== "GET" || request.destination !== "image") return response;
|
||||
let expired = false, timer;
|
||||
try {
|
||||
const repair = async () => {
|
||||
if (!request.url.startsWith(this.replayPrefix)) return response;
|
||||
const name = this.collections.root || request.url.slice(this.replayPrefix.length).split("/", 1)[0];
|
||||
const collection = await this.collections.getColl(name);
|
||||
// Do not operate on live/proxy collections or remote source archives.
|
||||
// lookupUrl is the archive index API, absent on the live proxy store.
|
||||
if (!collection || typeof collection.store?.lookupUrl !== "function" || collection.liveRedirectOnNotFound ||
|
||||
new URL(collection.config.sourceUrl).origin !== self.location.origin) return response;
|
||||
const missing = splitReplay(request.url, collection.prefix);
|
||||
const referring = splitReplay(request.referrer, collection.prefix);
|
||||
if (!missing || !referring) return response;
|
||||
const archiveURL = url => collection.prefix + referring[1] + "id_/" + url;
|
||||
const read = url => this.getResponseFor(new Request(archiveURL(url), {
|
||||
credentials: "same-origin", redirect: "manual", signal: request.signal,
|
||||
}), event);
|
||||
const key = archiveURL(referring[2]);
|
||||
if (!documents.has(key)) {
|
||||
// Cache only candidate metadata, never whole HTML or image bodies.
|
||||
// Both entry count and HTML reads are bounded; WACZ range loading
|
||||
// remains the upstream replay engine's responsibility.
|
||||
if (documents.size >= 16) documents.delete(documents.keys().next().value);
|
||||
documents.set(key, (async () => {
|
||||
const html = await read(referring[2]);
|
||||
if (html.status !== 200 || !html.headers.get("content-type")?.includes("text/html")) return [];
|
||||
const reader = html.body.getReader(), decoder = new TextDecoder();
|
||||
let text = "", size = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
size += chunk.value.byteLength;
|
||||
if (expired || size > 2 * 1024 * 1024) return [];
|
||||
text += decoder.decode(chunk.value, {stream:true});
|
||||
}
|
||||
text += decoder.decode();
|
||||
} finally {await reader.cancel().catch(() => {});}
|
||||
// Ignore tag-shaped strings in scripts/comments. Parsing is
|
||||
// intentionally conservative; an odd/malformed page stays broken
|
||||
// instead of acquiring an unrelated "similar looking" image.
|
||||
text = text.replace(/<!--[\s\S]*?-->|<(script|style)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, "");
|
||||
const groups = [];
|
||||
let base = referring[2], picture = null;
|
||||
const absolute = value => {
|
||||
try {
|
||||
const url = new URL(value, base);
|
||||
if (!/^https?:$/.test(url.protocol) || url.username || url.password) return null;
|
||||
url.hash = "";
|
||||
return url.href;
|
||||
} catch {return null;}
|
||||
};
|
||||
for (const token of text.matchAll(/<\/?(?:base|picture|source|img)\b(?:"[^"]*"|'[^']*'|[^'">])*>/gi)) {
|
||||
const tag = token[0], attrs = attributes(tag);
|
||||
if (/^<base\b/i.test(tag)) {if (attrs.href) base = absolute(attrs.href) || base; continue;}
|
||||
if (/^<picture\b/i.test(tag)) {picture = []; continue;}
|
||||
if (/^<\/picture\b/i.test(tag)) {picture = null; continue;}
|
||||
const urls = [...new Set([attrs.src, ...sourceSet(attrs.srcset)].filter(Boolean).map(absolute).filter(Boolean))];
|
||||
if (/^<source\b/i.test(tag)) {if (picture) picture.push(...urls); continue;}
|
||||
if (!attrs.srcset && !picture?.length) continue;
|
||||
groups.push([...new Set([...urls, ...(picture || [])])].slice(0, 64));
|
||||
if (groups.length >= 512) break;
|
||||
}
|
||||
return groups;
|
||||
})());
|
||||
}
|
||||
const groups = await documents.get(key);
|
||||
const failed = new URL(missing[2], referring[2]); failed.hash = "";
|
||||
const candidates = groups.find(group => group.includes(failed.href));
|
||||
if (!candidates) return response;
|
||||
// Prefer the img's explicit src, then alternatives in declaration
|
||||
// order. Limit extra archive lookups, and never follow redirects.
|
||||
for (const url of candidates.filter(url => url !== failed.href).slice(0, 6)) {
|
||||
if (expired) break;
|
||||
const alternative = await read(url);
|
||||
if (alternative.status !== 200 || !alternative.headers.get("content-type")?.startsWith("image/")) continue;
|
||||
if (expired) break;
|
||||
const headers = new Headers(alternative.headers);
|
||||
headers.set("X-ArchiveBox-Image-Fallback", url);
|
||||
headers.set("Cache-Control", "no-store");
|
||||
return new Response(alternative.body, {status:200, headers});
|
||||
}
|
||||
return response;
|
||||
};
|
||||
return await Promise.race([
|
||||
repair().catch(() => response),
|
||||
new Promise(resolve => {timer = setTimeout(() => {expired = true; resolve(response);}, 1500);}),
|
||||
]);
|
||||
} catch {return response;}
|
||||
finally {expired = true; clearTimeout(timer);}
|
||||
};
|
||||
replay.__abxImageFallback = true;
|
||||
} catch { /* Optional repair must never prevent the worker from starting. */ }
|
||||
})();
|
||||
"""
|
||||
try:
|
||||
body = response.content
|
||||
if b"self.sw=" not in body or b"__abxImageFallback" in body:
|
||||
return response
|
||||
response.content = body + script
|
||||
# Upstream headers describe the unmodified worker. Revalidate this
|
||||
# generated asset so existing captures can receive the repair too.
|
||||
for header in ("ETag", "Last-Modified", "Content-Length"):
|
||||
if response.has_header(header):
|
||||
del response[header]
|
||||
response["Cache-Control"] = "no-cache"
|
||||
except Exception:
|
||||
pass
|
||||
return response
|
||||
|
||||
|
||||
def _load_hash_map(snapshot_dir: Path) -> dict[str, str] | None:
|
||||
hashes_path = snapshot_dir / "hashes" / "hashes.json"
|
||||
if not hashes_path.exists():
|
||||
@@ -126,7 +313,8 @@ def _resolve_archive_path(document_root: str | Path, rel_path: str) -> tuple[Pat
|
||||
current = match
|
||||
resolved_parts.append(match.name)
|
||||
|
||||
return current, posixpath.join(*resolved_parts) if resolved_parts else ""
|
||||
resolved_path = posixpath.join(*resolved_parts) if resolved_parts else ""
|
||||
return Path(safe_join(document_root, resolved_path)), resolved_path
|
||||
|
||||
|
||||
def _cache_policy(config=None, **config_kwargs) -> str:
|
||||
@@ -308,10 +496,24 @@ def _render_directory_index(request, path: str, fullpath: Path) -> HttpResponse:
|
||||
file_list.append(url)
|
||||
|
||||
stat_result = entry.stat()
|
||||
preview_kind = ""
|
||||
mime = mimetypes.guess_type(entry.name)[0] or "application/octet-stream"
|
||||
if entry.is_file() and stat_result.st_size < DIRECTORY_PREVIEW_MAX_BYTES:
|
||||
if mime.startswith("image/"):
|
||||
preview_kind = "image"
|
||||
elif (
|
||||
mime.startswith("text/")
|
||||
or re.search(r"(?:^|[.+/-])(?:json|xml|javascript|ecmascript|yaml|toml|graphql|sql)(?:[.+/-]|$)", mime)
|
||||
or entry.suffix.lower() in {".log", ".jsonl", ".toml"}
|
||||
):
|
||||
preview_kind = "text"
|
||||
entries.append(
|
||||
{
|
||||
"name": url,
|
||||
"url": url,
|
||||
"preview_kind": preview_kind,
|
||||
"mimetype": "Directory" if entry.is_dir() else mime,
|
||||
"size_bytes": 0 if entry.is_dir() else stat_result.st_size,
|
||||
"is_dir": entry.is_dir(),
|
||||
"size": "—" if entry.is_dir() else printable_filesize(stat_result.st_size),
|
||||
"timestamp": _format_direntry_timestamp(stat_result),
|
||||
@@ -851,7 +1053,8 @@ def _is_risky_replay_document(fullpath: Path, content_type: str) -> bool:
|
||||
# Unknown archived response paths often have no extension. Sniff a small prefix
|
||||
# so one-domain no-JS mode still catches HTML/SVG documents.
|
||||
try:
|
||||
head = fullpath.read_bytes()[:4096].decode("utf-8", errors="ignore").lower()
|
||||
with fullpath.open("rb") as source:
|
||||
head = source.read(4096).decode("utf-8", errors="ignore").lower()
|
||||
except (OSError, UnicodeDecodeError):
|
||||
return False
|
||||
|
||||
@@ -1020,6 +1223,29 @@ def serve_static_with_byterange_support(request, path, document_root=None, show_
|
||||
config=config,
|
||||
)
|
||||
|
||||
# A shared archive explorer for ZIP outputs from any plugin. Raw/range
|
||||
# requests retain the original bytes for downloads and seekable reads.
|
||||
if fullpath.suffix.lower() == ".zip" and request.GET.get("preview") and not request.META.get("HTTP_RANGE"):
|
||||
raw_query = request.GET.copy()
|
||||
raw_query.pop("preview", None)
|
||||
raw_query["raw"] = "1"
|
||||
wrapped = loader.get_template("static/zip_index.html").render(
|
||||
{
|
||||
"title": fullpath.name,
|
||||
"output_path": f"{request.path}?{raw_query.urlencode()}",
|
||||
"archive_size": statobj.st_size,
|
||||
},
|
||||
)
|
||||
response = HttpResponse(wrapped, content_type="text/html; charset=utf-8")
|
||||
_set_transformed_response_headers(response, fullpath, statobj, None, cache_policy)
|
||||
return _apply_archive_replay_headers(
|
||||
response,
|
||||
fullpath=fullpath,
|
||||
content_type="text/html; charset=utf-8",
|
||||
is_archive_replay=is_archive_replay,
|
||||
config=config,
|
||||
)
|
||||
|
||||
# Wrap text-like outputs in HTML when explicitly requested for iframe previewing.
|
||||
if preview_as_text_html:
|
||||
try:
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""ArchiveBox OpenCode admin integration."""
|
||||
@@ -1,13 +0,0 @@
|
||||
from django.urls import path, re_path
|
||||
|
||||
from archivebox.opencode.views import agent_view, opencode_proxy_view
|
||||
|
||||
|
||||
urlpatterns = [
|
||||
path("", agent_view, name="opencode-agent"),
|
||||
re_path(
|
||||
r"^opencode(?:/(?P<path>.*))?$",
|
||||
opencode_proxy_view,
|
||||
name="opencode-proxy",
|
||||
),
|
||||
]
|
||||
@@ -1,140 +0,0 @@
|
||||
"""Django adapter for the optional plugin; never import its runtime at startup."""
|
||||
|
||||
import logging
|
||||
from io import BytesIO
|
||||
|
||||
from asgiref.sync import sync_to_async
|
||||
|
||||
from django.contrib.auth.middleware import AuthenticationMiddleware
|
||||
from django.contrib.auth.views import redirect_to_login
|
||||
from django.contrib.sessions.middleware import SessionMiddleware
|
||||
from django.core.exceptions import PermissionDenied
|
||||
from django.core.handlers.asgi import ASGIRequest
|
||||
from django.db import close_old_connections
|
||||
from django.http import Http404, HttpResponse, HttpResponseForbidden, StreamingHttpResponse
|
||||
from django.template import engines
|
||||
from django.urls import Resolver404, resolve
|
||||
from django.views.decorators.csrf import csrf_exempt
|
||||
|
||||
from archivebox.config import CONSTANTS
|
||||
from archivebox.config.common import get_request_config
|
||||
from archivebox.core.middleware import ReverseProxyAuthMiddleware
|
||||
from archivebox.core.routes_util import build_admin_url, get_admin_host, get_api_base_url, get_base_url, host_matches
|
||||
from archivebox.plugins.discovery import get_plugin_template
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _runtime_settings(request, config):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
settings = runtime._settings(config, CONSTANTS.DATA_DIR)
|
||||
route_config = request.__dict__.get("archivebox_config")
|
||||
settings.update(
|
||||
archivebox_base_url=get_base_url(request=request, config=route_config).rstrip("/"),
|
||||
archivebox_admin_url=build_admin_url("/admin/", request=request, config=route_config).rstrip("/"),
|
||||
archivebox_api_url=f"{get_api_base_url(request=request, config=route_config).rstrip('/')}/api/",
|
||||
)
|
||||
return runtime, settings
|
||||
|
||||
|
||||
def _dispatch(request, path=None):
|
||||
try:
|
||||
# Middleware already merged server defaults and live Machine overrides.
|
||||
# Re-resolving every extractor's config here stalls the UI request burst.
|
||||
config = get_request_config(request).model_dump(mode="json")
|
||||
if not config.get("OPENCODE_ENABLED", False):
|
||||
raise Http404
|
||||
if not request.user.is_authenticated:
|
||||
return redirect_to_login(request.get_full_path(), login_url="/admin/login/")
|
||||
if not request.user.is_active or not request.user.is_superuser:
|
||||
return HttpResponseForbidden("Agent access requires a superuser account.")
|
||||
|
||||
runtime, settings = _runtime_settings(request, config)
|
||||
if path is None:
|
||||
from archivebox.core.admin_site import archivebox_admin
|
||||
|
||||
context = {**archivebox_admin.each_context(request), **runtime.agent_context(settings)}
|
||||
source = get_plugin_template("opencode", "agent", fallback=False)
|
||||
if source is None:
|
||||
raise RuntimeError("Agent template unavailable")
|
||||
return HttpResponse(engines["django"].from_string(source).render(context, request))
|
||||
|
||||
if not runtime._origin_allowed(request.method, request.get_host(), request.headers):
|
||||
return HttpResponseForbidden("Cross-origin agent requests are blocked.")
|
||||
status, headers, body = runtime.proxy(
|
||||
settings,
|
||||
request.method,
|
||||
path,
|
||||
tuple((key, value) for key, values in request.GET.lists() for value in values),
|
||||
request.headers,
|
||||
request.body,
|
||||
)
|
||||
response_type = HttpResponse if isinstance(body, bytes) else StreamingHttpResponse
|
||||
response = response_type(body, status=status, headers=headers)
|
||||
response.xframe_options_exempt = True
|
||||
response.headers["X-Frame-Options"] = "SAMEORIGIN"
|
||||
response.headers["Content-Security-Policy"] = "frame-ancestors 'self'"
|
||||
return response
|
||||
except Http404:
|
||||
raise
|
||||
except Exception:
|
||||
# Optional-service boundary, including imports and template rendering.
|
||||
_LOGGER.exception("Optional AI service failed")
|
||||
return HttpResponse("AI service unavailable. See server logs.", status=503, content_type="text/plain")
|
||||
|
||||
|
||||
def agent_view(request):
|
||||
return _dispatch(request)
|
||||
|
||||
|
||||
@csrf_exempt
|
||||
def opencode_proxy_view(request, path=None):
|
||||
return _dispatch(request, path=path or "")
|
||||
|
||||
|
||||
def _websocket_context(scope):
|
||||
close_old_connections()
|
||||
try:
|
||||
match = resolve(scope["path"])
|
||||
if match.url_name != "opencode-proxy":
|
||||
raise PermissionDenied
|
||||
request = ASGIRequest(
|
||||
{**scope, "type": "http", "method": "GET", "scheme": "https" if scope.get("scheme") == "wss" else "http"},
|
||||
BytesIO(),
|
||||
)
|
||||
route_config = get_request_config(request, resolve_plugins=False)
|
||||
request.archivebox_config = route_config
|
||||
if not route_config.CONTROL_PLANE_ENABLED:
|
||||
raise PermissionDenied
|
||||
if (
|
||||
route_config.USES_SUBDOMAIN_ROUTING
|
||||
and route_config.BASE_URL
|
||||
and not host_matches(request.get_host(), get_admin_host(config=route_config, request=request))
|
||||
):
|
||||
raise PermissionDenied
|
||||
SessionMiddleware(_dispatch).process_request(request)
|
||||
AuthenticationMiddleware(_dispatch).process_request(request)
|
||||
ReverseProxyAuthMiddleware(_dispatch).process_request(request)
|
||||
config = route_config.model_dump(mode="json")
|
||||
if not config.get("OPENCODE_ENABLED") or not request.user.is_active or not request.user.is_superuser:
|
||||
raise PermissionDenied
|
||||
runtime, settings = _runtime_settings(request, config)
|
||||
if not request.headers.get("Origin") or not runtime._origin_allowed("POST", request.get_host(), request.headers):
|
||||
raise PermissionDenied
|
||||
return runtime, settings, match.kwargs.get("path", "")
|
||||
finally:
|
||||
close_old_connections()
|
||||
|
||||
|
||||
async def websocket_view(scope, receive, send):
|
||||
if (await receive())["type"] != "websocket.connect":
|
||||
return
|
||||
try:
|
||||
runtime, settings, path = await sync_to_async(_websocket_context)(scope)
|
||||
await runtime.websocket_proxy(settings, path, scope.get("query_string", b""), scope.get("subprotocols", []), receive, send)
|
||||
except (PermissionDenied, Resolver404, Http404):
|
||||
await send({"type": "websocket.close", "code": 1008})
|
||||
except Exception:
|
||||
_LOGGER.exception("Optional AI WebSocket failed")
|
||||
await send({"type": "websocket.close", "code": 1011})
|
||||
@@ -192,7 +192,9 @@ def serve_plugin_replay_asset(path: str, config, response_class):
|
||||
for module in get_plugin_presentation_modules():
|
||||
hook = getattr(module, "serve_replay_asset_response", None)
|
||||
if hook is not None and (response := hook(path, config, response_class)) is not None:
|
||||
return response
|
||||
from archivebox.misc.serve_static import add_replay_image_fallback
|
||||
|
||||
return add_replay_image_fallback(response)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -210,6 +210,12 @@ def _save_archiveresult_event_to_db(
|
||||
|
||||
with _perf_span("archivebox.ArchiveResultService.on_ArchiveResultEvent.diff_fields"):
|
||||
update_fields = []
|
||||
# This runner already loaded the Snapshot/Crawl for the event. Resolve
|
||||
# retention here without extra parent reads so new results do not wait
|
||||
# for the bounded legacy-repair sweep to traverse historical rows.
|
||||
result.snapshot = snapshot
|
||||
if result.set_delete_at_from_config():
|
||||
update_fields.append("delete_at")
|
||||
for field, value in defaults.items():
|
||||
if result.__dict__[field] != value:
|
||||
setattr(result, field, value)
|
||||
|
||||
@@ -170,6 +170,18 @@ class ResourceAdmission:
|
||||
used_bytes = 0
|
||||
for child in processes:
|
||||
try:
|
||||
if sys.platform.startswith("linux"):
|
||||
try:
|
||||
memory = child.memory_full_info()
|
||||
# Chrome's processes share substantial resident
|
||||
# memory. RSS charges those pages once per process
|
||||
# and can falsely serialize captures on an idle host.
|
||||
used_bytes += memory.pss + memory.swap
|
||||
continue
|
||||
except psutil.AccessDenied:
|
||||
# Restricted /proc access still gets conservative
|
||||
# accounting through the available RSS measurement.
|
||||
pass
|
||||
used_bytes += child.memory_info().rss
|
||||
except (psutil.Error, OSError):
|
||||
continue
|
||||
|
||||
@@ -12,6 +12,7 @@ import threading
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from datetime import timedelta
|
||||
from itertools import batched
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any, Literal
|
||||
@@ -1693,7 +1694,9 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
|
||||
# every URL, even though crawl isolation owns one browser for the whole
|
||||
# crawl. The crawl lifecycle lock already serializes all of its child
|
||||
# work, so let one runner drain the due siblings and discoveries.
|
||||
if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
|
||||
if any(
|
||||
process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)
|
||||
):
|
||||
snapshot.update_and_requeue(retry_at=timezone.now() + timedelta(seconds=lock_seconds))
|
||||
return True
|
||||
if snapshot.fs_migration_needed:
|
||||
@@ -1727,7 +1730,7 @@ def _run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interru
|
||||
if not snapshot.claim_processing_lock(lock_seconds=lock_seconds):
|
||||
return False
|
||||
snapshot.refresh_from_db()
|
||||
if any(process.is_running for process in snapshot.process_set.filter(status="running").iterator()):
|
||||
if any(process.is_running for process in snapshot.process_set.filter(status="running").order_by("pk").paged_iterator(chunk_size=100)):
|
||||
# The Snapshot lease may have expired while an abx-dl hook process is
|
||||
# still alive. Preserve the snapshot-level ownership boundary and do
|
||||
# not launch a second sequence; ArchiveResult status is irrelevant.
|
||||
@@ -1901,7 +1904,24 @@ def run_install(*, plugin_names: list[str] | None = None) -> None:
|
||||
|
||||
def _first_due_id(queryset, *, newest_first: bool = False):
|
||||
ordering = ("-retry_at", "-created_at") if newest_first else ("retry_at", "created_at")
|
||||
return queryset.order_by(*ordering).values_list("id", flat=True).first()
|
||||
# Queue membership comes from retry_at, not status: almost the entire
|
||||
# archive can be sealed while only a handful of rows need maintenance.
|
||||
# Combining both predicates lets SQLite choose the broad status index and
|
||||
# scan/sort archival history on every idle pass (567ms on Cabbage vs 3ms
|
||||
# through retry_at). First materialize bounded due IDs using the existing
|
||||
# queue index, then apply status/parent filters to those IDs. This also
|
||||
# bounds PostgreSQL reads and keeps every cursor closed before a CAS claim.
|
||||
due_ids = (
|
||||
queryset.model.objects.filter(retry_at__lte=timezone.now())
|
||||
.order_by(*ordering)
|
||||
.values_list("id", flat=True)
|
||||
.paged_iterator(chunk_size=100)
|
||||
)
|
||||
for candidate_ids in batched(due_ids, 100):
|
||||
match = queryset.filter(id__in=candidate_ids).order_by(*ordering).values_list("id", flat=True).first()
|
||||
if match is not None:
|
||||
return match
|
||||
return None
|
||||
|
||||
|
||||
def _run_due_crawl_status(
|
||||
@@ -1972,12 +1992,8 @@ def _run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrup
|
||||
def _run_due_binary() -> bool:
|
||||
from archivebox.machine.models import Binary
|
||||
|
||||
due_binary_id = (
|
||||
Binary.objects.filter(retry_at__lte=timezone.now())
|
||||
.exclude(status=Binary.StatusChoices.INSTALLED)
|
||||
.order_by("retry_at", "created_at")
|
||||
.values_list("id", flat=True)
|
||||
.first()
|
||||
due_binary_id = _first_due_id(
|
||||
Binary.objects.filter(retry_at__lte=timezone.now()).exclude(status=Binary.StatusChoices.INSTALLED),
|
||||
)
|
||||
if due_binary_id is None:
|
||||
return False
|
||||
@@ -2004,6 +2020,7 @@ def run_pending_crawls(
|
||||
last_recovery_at = 0.0
|
||||
last_retention_at = 0.0
|
||||
last_retention_repair_at = 0.0
|
||||
retention_backfill_cursors: dict = {}
|
||||
deletion_thread: threading.Thread | None = None
|
||||
last_analyze_at = 0.0
|
||||
analyze_queue: list[str] | None = None
|
||||
@@ -2200,7 +2217,7 @@ def run_pending_crawls(
|
||||
# delete_at in the plugin-result hot path. Running it here keeps
|
||||
# DELETE_AFTER resolution fresh without making every hook event
|
||||
# load parent Snapshot/Crawl config.
|
||||
model.delete_expired(batch_size=100, backfill_missing=True)
|
||||
model.delete_expired(batch_size=100, backfill_missing=True, backfill_cursors=retention_backfill_cursors)
|
||||
last_retention_repair_at = now_monotonic
|
||||
|
||||
if daemon:
|
||||
@@ -2216,12 +2233,10 @@ def run_pending_crawls(
|
||||
# snapshot detail page out to ~500ms. Refresh stats at most once per
|
||||
# 24hr while the queue is idle, and only after the orchestrator has
|
||||
# been alive for at least an hour so short server boots / one-off work
|
||||
# never pay the cost. The sweep is batched one table per idle tick;
|
||||
# individual table ANALYZE statements abort after 2min (progress
|
||||
# handler) and the whole sweep is hard-capped at 5min so a
|
||||
# pathological table cannot wedge maintenance forever. Any failure
|
||||
# inside the maintenance hook is swallowed — orchestrator must never
|
||||
# be taken down by stats refresh.
|
||||
# never pay the cost. Each idle tick samples a bounded number of
|
||||
# index entries for one table; never scan the full history while
|
||||
# holding ANALYZE's write transaction. Optional stats refresh must
|
||||
# not prevent queued captures.
|
||||
try:
|
||||
if (
|
||||
analyze_queue is None
|
||||
|
||||
@@ -1,357 +1,177 @@
|
||||
{% load i18n %}
|
||||
{% get_current_language as language_code %}
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<html lang="{{ language_code }}">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
<meta http-equiv="Content-Language" content="en-us">
|
||||
<meta http-equiv="Content-Language" content="{{ language_code }}">
|
||||
<meta name="robots" content="NONE,NOARCHIVE">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{% blocktranslate %}Index of {{ directory }}{% endblocktranslate %}</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f3f5f8;
|
||||
--panel: #ffffff;
|
||||
--panel-border: #d9e0e8;
|
||||
--text: #16202a;
|
||||
--muted: #5e6b78;
|
||||
--accent: #0f5cc0;
|
||||
--accent-soft: #e7f0ff;
|
||||
--folder-bg: #f6f9ff;
|
||||
--folder-border: #d6e5ff;
|
||||
--shadow: 0 10px 30px rgba(20, 32, 52, 0.08);
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html,
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: linear-gradient(180deg, #eef3f9 0%, var(--bg) 100%);
|
||||
color: var(--text);
|
||||
font-family:
|
||||
ui-sans-serif,
|
||||
system-ui,
|
||||
-apple-system,
|
||||
BlinkMacSystemFont,
|
||||
"Segoe UI",
|
||||
sans-serif;
|
||||
}
|
||||
|
||||
body {
|
||||
min-height: 100vh;
|
||||
padding: 24px;
|
||||
}
|
||||
|
||||
.directory-shell {
|
||||
width: min(980px, 100%);
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.directory-toolbar {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
margin-bottom: 14px;
|
||||
padding: 12px;
|
||||
border: 1px solid var(--panel-border);
|
||||
border-radius: 16px;
|
||||
background: rgba(255, 255, 255, 0.9);
|
||||
box-shadow: var(--shadow);
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
|
||||
.toolbar-link {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
min-height: 40px;
|
||||
padding: 0 14px;
|
||||
border: 1px solid var(--panel-border);
|
||||
border-radius: 999px;
|
||||
background: #fff;
|
||||
color: var(--text);
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
text-decoration: none;
|
||||
transition:
|
||||
background-color 0.15s ease,
|
||||
border-color 0.15s ease,
|
||||
transform 0.15s ease;
|
||||
}
|
||||
|
||||
.toolbar-link:hover {
|
||||
background: #f8fbff;
|
||||
border-color: #c5d2e0;
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.toolbar-link.primary {
|
||||
background: var(--accent-soft);
|
||||
border-color: var(--folder-border);
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.archivebox-zip-spinner {
|
||||
display: none;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
border: 2px solid #2563eb;
|
||||
border-right-color: transparent;
|
||||
border-radius: 999px;
|
||||
animation: archivebox-zip-spin 0.75s linear infinite;
|
||||
flex: 0 0 auto;
|
||||
box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.45);
|
||||
}
|
||||
|
||||
.archivebox-zip-button.is-loading {
|
||||
pointer-events: none;
|
||||
opacity: 0.96;
|
||||
}
|
||||
|
||||
.archivebox-zip-button.is-loading .archivebox-zip-spinner {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.archivebox-zip-button.is-loading .archivebox-zip-label {
|
||||
opacity: 0.72;
|
||||
}
|
||||
|
||||
@keyframes archivebox-zip-spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
.directory-card {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--panel-border);
|
||||
border-radius: 18px;
|
||||
box-shadow: var(--shadow);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.directory-header {
|
||||
padding: 22px 24px 18px;
|
||||
border-bottom: 1px solid var(--panel-border);
|
||||
background:
|
||||
radial-gradient(circle at top right, rgba(15, 92, 192, 0.08), transparent 28%),
|
||||
linear-gradient(180deg, #ffffff, #f8fbff);
|
||||
}
|
||||
|
||||
.directory-kicker {
|
||||
margin: 0 0 8px;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.08em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.directory-title {
|
||||
margin: 0;
|
||||
font-size: clamp(22px, 4vw, 34px);
|
||||
line-height: 1.1;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.directory-meta {
|
||||
margin-top: 12px;
|
||||
color: var(--muted);
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.directory-path {
|
||||
display: inline-block;
|
||||
margin-top: 12px;
|
||||
padding: 8px 12px;
|
||||
border-radius: 999px;
|
||||
background: #f4f7fb;
|
||||
border: 1px solid var(--panel-border);
|
||||
color: #304152;
|
||||
font-family:
|
||||
ui-monospace,
|
||||
"SFMono-Regular",
|
||||
Menlo,
|
||||
Consolas,
|
||||
monospace;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.directory-list {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.directory-entry + .directory-entry {
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.directory-link {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
width: 100%;
|
||||
padding: 12px 14px;
|
||||
border: 1px solid transparent;
|
||||
border-radius: 12px;
|
||||
color: inherit;
|
||||
text-decoration: none;
|
||||
transition:
|
||||
background-color 0.15s ease,
|
||||
border-color 0.15s ease,
|
||||
transform 0.15s ease,
|
||||
box-shadow 0.15s ease;
|
||||
}
|
||||
|
||||
.directory-link:hover {
|
||||
background: #f8fbff;
|
||||
border-color: var(--panel-border);
|
||||
transform: translateY(-1px);
|
||||
}
|
||||
|
||||
.directory-link:focus-visible {
|
||||
outline: 3px solid rgba(15, 92, 192, 0.2);
|
||||
outline-offset: 1px;
|
||||
}
|
||||
|
||||
.directory-entry.folder .directory-link {
|
||||
background: var(--folder-bg);
|
||||
border-color: var(--folder-border);
|
||||
}
|
||||
|
||||
.entry-icon {
|
||||
flex: 0 0 auto;
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border-radius: 10px;
|
||||
background: #edf2f8;
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
.directory-entry.folder .entry-icon {
|
||||
background: var(--accent-soft);
|
||||
}
|
||||
|
||||
.entry-label {
|
||||
min-width: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
}
|
||||
|
||||
.entry-name {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
line-height: 1.35;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.entry-kind {
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
.directory-entry.parent .entry-kind {
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
padding: 28px 24px 32px;
|
||||
text-align: center;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
body {
|
||||
padding: 14px;
|
||||
}
|
||||
|
||||
.directory-header {
|
||||
padding: 18px 18px 16px;
|
||||
}
|
||||
|
||||
.directory-list {
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.directory-link {
|
||||
padding: 11px 12px;
|
||||
}
|
||||
}
|
||||
:root { color-scheme: light; --line: #dfe4eb; --text: #253247; --muted: #667085; --accent: #245dc1; }
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; padding: 0; min-height: 100%; background: white; color: var(--text); font: 13px/1.4 system-ui, sans-serif; }
|
||||
main { width: 100%; }
|
||||
a { color: inherit; text-decoration: none; }
|
||||
.directory-toolbar { display: flex; flex-wrap: wrap; gap: 6px; padding: 8px 12px; border-bottom: 1px solid var(--line); }
|
||||
.toolbar-link, .entry-download { display: inline-flex; align-items: center; justify-content: center; gap: 5px; height: 30px; padding: 4px 9px; border: 1px solid var(--line); border-radius: 4px; background: white; }
|
||||
.toolbar-link:hover, .entry-download:hover { background: #f0f5ff; }
|
||||
.directory-header { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; padding: 8px 12px; border-bottom: 1px solid var(--line); background: #f8fafc; }
|
||||
.directory-title { margin: 0; font-size: 13px; font-weight: 600; overflow-wrap: anywhere; }
|
||||
.directory-meta { margin-left: auto; color: var(--muted); white-space: nowrap; }
|
||||
#filter-files { min-width: 0; width: 180px; padding: 5px 8px; border: 1px solid var(--line); border-radius: 4px; font: inherit; }
|
||||
.directory-list { width: 100%; border-collapse: collapse; table-layout: fixed; }
|
||||
th { height: 30px; padding: 0; text-align: left; background: #f8fafc; border-bottom: 1px solid var(--line); }
|
||||
th:first-child button { padding-left: 72px; }
|
||||
th button { width: 100%; height: 30px; padding: 0 8px; border: 0; background: none; color: var(--muted); text-align: left; font: inherit; cursor: pointer; }
|
||||
th[aria-sort="ascending"] button::after { content: " ↑"; }
|
||||
th[aria-sort="descending"] button::after { content: " ↓"; }
|
||||
.size-column { width: 90px; }
|
||||
.type-column { width: 220px; }
|
||||
.download-column { width: 50px; }
|
||||
td { padding: 0; }
|
||||
.entry-size, .entry-type { padding: 0 8px; color: var(--muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.entry-size { font-variant-numeric: tabular-nums; }
|
||||
.directory-entry { height: 50px; border-bottom: 1px solid var(--line); }
|
||||
.directory-entry[hidden] { display: none; }
|
||||
.directory-entry:hover { background: #f8fafc; }
|
||||
.directory-link { display: grid; grid-template-columns: 48px minmax(0, 1fr); align-items: center; gap: 12px; min-width: 0; height: 49px; padding: 6px 12px; }
|
||||
a:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; }
|
||||
.entry-icon, .entry-preview { width: 48px; height: 36px; margin: 0; overflow: hidden; border-radius: 3px; }
|
||||
.entry-icon { display: grid; place-items: center; font-size: 20px; }
|
||||
.entry-preview { border: 1px solid var(--line); background: white; }
|
||||
img.entry-preview { object-fit: contain; }
|
||||
pre.entry-preview { padding: 3px; white-space: pre-wrap; overflow-wrap: anywhere; font: 7px/1.2 ui-monospace, monospace; color: var(--muted); }
|
||||
.entry-label { min-width: 0; display: flex; flex-direction: column; gap: 1px; }
|
||||
.entry-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-weight: 500; }
|
||||
.entry-kind { color: var(--muted); font-size: 11px; }
|
||||
.entry-download { flex-shrink: 0; color: var(--accent); }
|
||||
.empty-state { padding: 24px; text-align: center; color: var(--muted); }
|
||||
.archivebox-zip-spinner { display: none; width: 12px; height: 12px; border: 2px solid var(--accent); border-right-color: transparent; border-radius: 50%; animation: archivebox-zip-spin .75s linear infinite; }
|
||||
.archivebox-zip-button.is-loading { pointer-events: none; opacity: .7; }
|
||||
.archivebox-zip-button.is-loading .archivebox-zip-spinner { display: inline-block; }
|
||||
@keyframes archivebox-zip-spin { to { transform: rotate(360deg); } }
|
||||
@media (max-width: 640px) { .size-column { width: 65px; } .type-column { width: 100px; } .download-column { width: 40px; } #filter-files { width: 130px; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="directory-shell">
|
||||
<nav class="directory-toolbar" aria-label="Directory navigation">
|
||||
<main>
|
||||
<nav class="directory-toolbar" aria-label="{% translate "Directory navigation" %}">
|
||||
{% if directory != "/" %}
|
||||
<a class="toolbar-link" href="../{{ directory_query }}">↩ Up One Level</a>
|
||||
<a class="toolbar-link" href="../{{ directory_query }}">↩ {% translate "Up One Level" %}</a>
|
||||
{% endif %}
|
||||
<a class="toolbar-link primary" href="{{ snapshot_page_url }}">⌂ Snapshot Page</a>
|
||||
<a class="toolbar-link" href="{{ snapshot_page_url }}">⌂ {% translate "Snapshot Page" %}</a>
|
||||
<a
|
||||
class="toolbar-link archivebox-zip-button"
|
||||
href="{{ zip_url }}"
|
||||
data-loading-label="Preparing..."
|
||||
data-loading-label="{% translate "Preparing..." %}"
|
||||
onclick="return window.archiveboxHandleZipClick(this, event);"
|
||||
>
|
||||
<span class="archivebox-zip-spinner" aria-hidden="true"></span>
|
||||
<span class="archivebox-zip-label">⬇ Download Zip</span>
|
||||
<span class="archivebox-zip-label">⬇ {% translate "Download Zip" %}</span>
|
||||
</a>
|
||||
</nav>
|
||||
|
||||
<section class="directory-card">
|
||||
<section>
|
||||
<header class="directory-header">
|
||||
<p class="directory-kicker">ArchiveBox File Index</p>
|
||||
<h1 class="directory-title">{% blocktranslate %}Index of {{ directory }}{% endblocktranslate %}</h1>
|
||||
<div class="directory-meta">
|
||||
{{ file_list|length }} item{{ file_list|length|pluralize }}
|
||||
<h1 class="directory-title">{{ directory }}</h1>
|
||||
<div class="directory-meta" id="file-count" aria-live="polite">
|
||||
{% translate "Items" %}: <span id="visible-file-count">{{ file_list|length }}</span>
|
||||
</div>
|
||||
<div class="directory-path">{{ directory }}</div>
|
||||
<input id="filter-files" type="search" aria-label="{% translate "Filter files" %}" placeholder="{% translate "Filter files" %}">
|
||||
</header>
|
||||
|
||||
{% if file_list or directory != "/" %}
|
||||
<ul class="directory-list">
|
||||
<table class="directory-list">
|
||||
<colgroup><col><col class="size-column"><col class="type-column"><col class="download-column"></colgroup>
|
||||
<thead><tr>
|
||||
<th scope="col" aria-sort="ascending"><button type="button" data-sort="name" aria-label="{% translate "Name" %}">{% translate "Name" %}</button></th>
|
||||
<th scope="col" aria-sort="none"><button type="button" data-sort="size" aria-label="{% translate "Size" %}">{% translate "Size" %}</button></th>
|
||||
<th scope="col" aria-sort="none"><button type="button" data-sort="type" aria-label="{% translate "Type" %}">{% translate "Type" %}</button></th>
|
||||
<th scope="col" aria-label="{% translate "Download" %}"></th>
|
||||
</tr></thead>
|
||||
<tbody>
|
||||
{% if directory != "/" %}
|
||||
<li class="directory-entry parent">
|
||||
<tr class="directory-entry parent"><td>
|
||||
<a class="directory-link" href="../{{ directory_query }}">
|
||||
<span class="entry-icon" aria-hidden="true">↩</span>
|
||||
<span class="entry-label">
|
||||
<span class="entry-name">../</span>
|
||||
<span class="entry-kind">Parent directory</span>
|
||||
</span>
|
||||
</a>
|
||||
</li>
|
||||
</td><td class="entry-size">—</td><td class="entry-type">{% translate "Directory" %}</td><td></td></tr>
|
||||
{% endif %}
|
||||
|
||||
{% for f in file_list %}
|
||||
<li class="directory-entry{% if f|slice:'-1:' == '/' %} folder{% endif %}">
|
||||
<a class="directory-link" href="{{ f|urlencode }}{% if f|slice:'-1:' == '/' %}{{ directory_query }}{% endif %}">
|
||||
<span class="entry-icon" aria-hidden="true">{% if f|slice:'-1:' == '/' %}📁{% else %}📄{% endif %}</span>
|
||||
{% for entry in entries %}
|
||||
{% with f=entry.name %}
|
||||
<tr class="directory-entry{% if entry.is_dir %} folder{% endif %}" data-name="{{ f }}" data-size="{{ entry.size_bytes }}" data-type="{{ entry.mimetype }}"><td>
|
||||
<a class="directory-link" href="{{ f|urlencode }}{% if f|slice:'-1:' == '/' %}{{ directory_query }}{% elif f|lower|slice:'-4:' == '.zip' %}?preview=1{% endif %}">
|
||||
{% if entry.preview_kind == "image" %}
|
||||
<img class="entry-preview" src="{{ f|urlencode }}?raw=1" alt="" loading="lazy" decoding="async">
|
||||
{% elif entry.preview_kind == "text" %}
|
||||
<pre class="entry-preview" data-url="{{ f|urlencode }}?raw=1" aria-hidden="true">…</pre>
|
||||
{% else %}
|
||||
<span class="entry-icon" aria-hidden="true">{% if entry.is_dir %}📁{% else %}📄{% endif %}</span>
|
||||
{% endif %}
|
||||
<span class="entry-label">
|
||||
<span class="entry-name">{{ f }}</span>
|
||||
<span class="entry-kind">{% if f|slice:'-1:' == '/' %}Directory{% else %}File{% endif %}</span>
|
||||
<span class="entry-name" title="{{ f }}">{{ f }}</span>
|
||||
</span>
|
||||
</a>
|
||||
</li>
|
||||
</td><td class="entry-size">{{ entry.size }}</td><td class="entry-type" title="{{ entry.mimetype }}">{{ entry.mimetype }}</td><td>
|
||||
<a class="entry-download" aria-label="{% blocktranslate %}Download {{ f }}{% endblocktranslate %}" title="{% blocktranslate %}Download {{ f }}{% endblocktranslate %}" href="{{ f|urlencode }}{% if entry.is_dir %}?files=1&download=zip{% else %}?raw=1{% endif %}" download>⬇</a>
|
||||
</td></tr>
|
||||
{% endwith %}
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<div class="empty-state">This directory is empty.</div>
|
||||
<div class="empty-state">{% translate "This directory is empty." %}</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
document.getElementById("filter-files").addEventListener("input", (event) => {
|
||||
const query = event.target.value.toLowerCase();
|
||||
let count = 0;
|
||||
document.querySelectorAll(".directory-entry:not(.parent)").forEach((row) => {
|
||||
row.hidden = !row.querySelector(".entry-name").textContent.toLowerCase().includes(query);
|
||||
if (!row.hidden) count++;
|
||||
});
|
||||
document.getElementById("visible-file-count").textContent = count;
|
||||
});
|
||||
let sortKey = null, sortDirection = 1;
|
||||
document.querySelectorAll("[data-sort]").forEach((button) => {
|
||||
button.addEventListener("click", () => {
|
||||
sortDirection = sortKey === button.dataset.sort ? -sortDirection : 1;
|
||||
sortKey = button.dataset.sort;
|
||||
const rows = [...document.querySelectorAll(".directory-entry:not(.parent)")];
|
||||
rows.sort((a, b) => {
|
||||
const folders = Number(b.classList.contains("folder")) - Number(a.classList.contains("folder"));
|
||||
if (folders) return folders;
|
||||
const comparison = sortKey === "size"
|
||||
? Number(a.dataset.size) - Number(b.dataset.size)
|
||||
: a.dataset[sortKey].localeCompare(b.dataset[sortKey], undefined, {numeric: true, sensitivity: "base"});
|
||||
return sortDirection * comparison || a.dataset.name.localeCompare(b.dataset.name, undefined, {numeric: true});
|
||||
});
|
||||
rows.forEach((row) => row.parentElement.appendChild(row));
|
||||
document.querySelectorAll("th[aria-sort]").forEach((header) => header.setAttribute("aria-sort", "none"));
|
||||
button.parentElement.setAttribute("aria-sort", sortDirection === 1 ? "ascending" : "descending");
|
||||
});
|
||||
});
|
||||
document.querySelector('[data-sort="name"]')?.click();
|
||||
const previews = new IntersectionObserver((items) => {
|
||||
for (const item of items) {
|
||||
if (!item.isIntersecting) continue;
|
||||
const preview = item.target;
|
||||
previews.unobserve(preview);
|
||||
fetch(preview.dataset.url)
|
||||
.then((response) => { if (!response.ok) throw new Error("Unavailable"); return response.text(); })
|
||||
.then((text) => { preview.textContent = text.slice(0, 12000); })
|
||||
.catch(() => { preview.textContent = "📄"; });
|
||||
}
|
||||
}, {rootMargin: "100px"});
|
||||
document.querySelectorAll("pre.entry-preview").forEach((preview) => previews.observe(preview));
|
||||
window.archiveboxHandleZipClick = function(link, event) {
|
||||
if (!link || link.dataset.loading === "1") {
|
||||
if (event) event.preventDefault();
|
||||
@@ -365,7 +185,7 @@
|
||||
const spinner = link.querySelector(".archivebox-zip-spinner");
|
||||
if (label) {
|
||||
link.dataset.originalLabel = label.textContent || "";
|
||||
label.textContent = link.dataset.loadingLabel || "Preparing...";
|
||||
label.textContent = link.dataset.loadingLabel;
|
||||
label.style.opacity = "0.92";
|
||||
}
|
||||
if (spinner) {
|
||||
|
||||
@@ -0,0 +1,792 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1" />
|
||||
<title>Saved files</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--line: #dfe4eb;
|
||||
--muted: #667085;
|
||||
--blue: #245dc1;
|
||||
}
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
html,
|
||||
body {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
}
|
||||
body {
|
||||
font: 13px/1.5 system-ui, sans-serif;
|
||||
color: #253247;
|
||||
background: white;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
a {
|
||||
color: var(--blue);
|
||||
text-decoration: none;
|
||||
}
|
||||
button,
|
||||
input {
|
||||
font: inherit;
|
||||
}
|
||||
button,
|
||||
a.action {
|
||||
cursor: pointer;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 5px;
|
||||
background: white;
|
||||
padding: 5px 10px;
|
||||
color: #344054;
|
||||
}
|
||||
button:hover,
|
||||
a.action:hover {
|
||||
background: #f0f5ff;
|
||||
}
|
||||
header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 10px 16px;
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
h1 {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
margin: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
nav {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
margin-left: auto;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.toolbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 9px 16px;
|
||||
background: #f8fafc;
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
#breadcrumbs {
|
||||
display: flex;
|
||||
gap: 5px;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
min-width: 0;
|
||||
}
|
||||
#breadcrumbs button {
|
||||
border: 0;
|
||||
background: none;
|
||||
padding: 0;
|
||||
color: var(--blue);
|
||||
}
|
||||
#count {
|
||||
color: var(--muted);
|
||||
white-space: nowrap;
|
||||
margin-left: auto;
|
||||
}
|
||||
#search {
|
||||
width: 160px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 5px;
|
||||
padding: 4px 8px;
|
||||
}
|
||||
main {
|
||||
display: flex;
|
||||
min-height: 0;
|
||||
flex: 1;
|
||||
}
|
||||
.listing {
|
||||
overflow: auto;
|
||||
flex: 1;
|
||||
min-width: 260px;
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
text-align: left;
|
||||
}
|
||||
th {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
background: #fff;
|
||||
color: var(--muted);
|
||||
font-weight: 500;
|
||||
border-bottom: 1px solid var(--line);
|
||||
padding: 8px 16px;
|
||||
}
|
||||
td {
|
||||
border-bottom: 1px solid #eef1f5;
|
||||
padding: 8px 16px;
|
||||
}
|
||||
td.name {
|
||||
width: 100%;
|
||||
}
|
||||
.entry {
|
||||
border: 0;
|
||||
background: none;
|
||||
padding: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
color: #253247;
|
||||
text-align: left;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.entry:hover {
|
||||
background: none;
|
||||
color: var(--blue);
|
||||
}
|
||||
.size,
|
||||
.date {
|
||||
color: var(--muted);
|
||||
white-space: nowrap;
|
||||
font-size: 12px;
|
||||
}
|
||||
tr.selected {
|
||||
background: #eaf2ff;
|
||||
}
|
||||
tr:hover {
|
||||
background: #f6f9fc;
|
||||
}
|
||||
#preview {
|
||||
width: 55%;
|
||||
min-width: 280px;
|
||||
border-left: 1px solid var(--line);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
min-height: 0;
|
||||
}
|
||||
#preview[hidden] {
|
||||
display: none;
|
||||
}
|
||||
.preview-head {
|
||||
padding: 9px 12px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
border-bottom: 1px solid var(--line);
|
||||
background: #f8fafc;
|
||||
}
|
||||
#filename {
|
||||
overflow-wrap: anywhere;
|
||||
font-weight: 600;
|
||||
margin-right: auto;
|
||||
}
|
||||
#content {
|
||||
overflow: auto;
|
||||
padding: 16px;
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: center;
|
||||
}
|
||||
#content img {
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
#content pre {
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
align-self: stretch;
|
||||
margin: 0;
|
||||
width: 100%;
|
||||
font: 12px/1.6 ui-monospace, monospace;
|
||||
}
|
||||
#content iframe {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border: 0;
|
||||
}
|
||||
#content audio,
|
||||
#content video {
|
||||
max-width: 100%;
|
||||
}
|
||||
.empty {
|
||||
padding: 22px;
|
||||
color: var(--muted);
|
||||
}
|
||||
#error {
|
||||
color: #b42318;
|
||||
padding: 8px 16px;
|
||||
margin: 0;
|
||||
}
|
||||
#error:empty {
|
||||
display: none;
|
||||
}
|
||||
@media (max-width: 640px) {
|
||||
header {
|
||||
flex-wrap: wrap;
|
||||
padding: 9px 12px;
|
||||
}
|
||||
nav {
|
||||
margin-left: 0;
|
||||
}
|
||||
.toolbar {
|
||||
padding: 8px 12px;
|
||||
}
|
||||
#search {
|
||||
width: 100px;
|
||||
}
|
||||
.date {
|
||||
display: none;
|
||||
}
|
||||
main {
|
||||
flex-direction: column;
|
||||
}
|
||||
.listing {
|
||||
min-height: 140px;
|
||||
}
|
||||
#preview {
|
||||
width: 100%;
|
||||
border-left: 0;
|
||||
border-top: 1px solid var(--line);
|
||||
min-height: 300px;
|
||||
flex: 1;
|
||||
}
|
||||
td,
|
||||
th {
|
||||
padding: 8px 12px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body data-output="{{ output_path }}" data-size="{{ archive_size }}">
|
||||
<header>
|
||||
<h1 id="title">{{ title }}</h1>
|
||||
<nav id="actions" aria-label="Output actions">
|
||||
<a class="action" id="files">View all files</a>
|
||||
</nav>
|
||||
</header>
|
||||
<div class="toolbar">
|
||||
<div id="breadcrumbs"></div>
|
||||
<span id="count"></span
|
||||
><input
|
||||
id="search"
|
||||
aria-label="Filter files"
|
||||
placeholder="Filter files"
|
||||
/>
|
||||
</div>
|
||||
<p id="error" role="alert"></p>
|
||||
<main>
|
||||
<section class="listing" aria-label="Files">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Size</th>
|
||||
<th class="date">Modified</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="entries"></tbody>
|
||||
</table>
|
||||
</section>
|
||||
<section id="preview" aria-label="File preview" hidden>
|
||||
<div class="preview-head">
|
||||
<span id="filename"></span><button id="download">Download</button
|
||||
><button id="close" aria-label="Close preview">×</button>
|
||||
</div>
|
||||
<div id="content"></div>
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
(() => {
|
||||
const byId = (id) => document.getElementById(id),
|
||||
el = (tag, text, cls) => {
|
||||
const n = document.createElement(tag);
|
||||
if (text != null) n.textContent = text;
|
||||
if (cls) n.className = cls;
|
||||
return n;
|
||||
};
|
||||
const output = new URL(document.body.dataset.output, document.baseURI);
|
||||
output.search = "";
|
||||
output.hash = "";
|
||||
const folder = new URL("./", output),
|
||||
raw = new URL(output);
|
||||
raw.searchParams.set("raw", "1");
|
||||
byId("files").href = folder.href + "?files=1";
|
||||
const MAX_PREVIEW = 16 * 1024 * 1024,
|
||||
CHUNK = 256 * 1024,
|
||||
decoder = new TextDecoder();
|
||||
const bytes = (n) =>
|
||||
n < 1024
|
||||
? n + " B"
|
||||
: n < 1048576
|
||||
? (n / 1024).toFixed(1) + " KB"
|
||||
: n < 1073741824
|
||||
? (n / 1048576).toFixed(1) + " MB"
|
||||
: (n / 1073741824).toFixed(2) + " GB";
|
||||
const safePath = (name) => {
|
||||
const p = name.replace(/\\/g, "/").replace(/\/$/, "");
|
||||
return p &&
|
||||
!p.startsWith("/") &&
|
||||
!p.split("/").some((x) => !x || x === "." || x === "..")
|
||||
? p
|
||||
: null;
|
||||
};
|
||||
const view = (b) => new DataView(b.buffer, b.byteOffset, b.byteLength);
|
||||
const u64 = (v, p) => {
|
||||
const n = v.getBigUint64(p, true);
|
||||
if (n > BigInt(Number.MAX_SAFE_INTEGER))
|
||||
throw Error("ZIP offsets exceed browser precision");
|
||||
return Number(n);
|
||||
};
|
||||
async function rangeResponse(archive, start, length, signal) {
|
||||
if (
|
||||
!Number.isSafeInteger(start) ||
|
||||
!Number.isSafeInteger(length) ||
|
||||
start < 0 ||
|
||||
length <= 0 ||
|
||||
start + length > archive.size
|
||||
)
|
||||
throw Error("Invalid ZIP byte range");
|
||||
const response = await fetch(archive.url, {
|
||||
headers: { Range: `bytes=${start}-${start + length - 1}` },
|
||||
signal,
|
||||
});
|
||||
const expected = `bytes ${start}-${start + length - 1}/${
|
||||
archive.size
|
||||
}`;
|
||||
if (
|
||||
response.status !== 206 ||
|
||||
response.headers.get("Content-Range") !== expected
|
||||
) {
|
||||
await response.body?.cancel();
|
||||
throw Error("ZIP browsing requires HTTP byte-range support");
|
||||
}
|
||||
return response;
|
||||
}
|
||||
async function range(archive, start, length, signal) {
|
||||
const response = await rangeResponse(archive, start, length, signal),
|
||||
reader = response.body.getReader(),
|
||||
result = new Uint8Array(length);
|
||||
let offset = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (offset + value.length > length)
|
||||
throw Error("ZIP range exceeds requested size");
|
||||
result.set(value, offset);
|
||||
offset += value.length;
|
||||
}
|
||||
if (offset !== length) throw Error("Incomplete ZIP byte range");
|
||||
return result;
|
||||
} finally {
|
||||
await reader.cancel();
|
||||
}
|
||||
}
|
||||
async function indexZip(archive) {
|
||||
const tail = await range(
|
||||
archive,
|
||||
Math.max(0, archive.size - 65557),
|
||||
Math.min(archive.size, 65557)
|
||||
);
|
||||
const v = view(tail);
|
||||
let end = -1;
|
||||
for (let i = tail.length - 22; i >= 0; i--)
|
||||
if (
|
||||
v.getUint32(i, true) === 0x06054b50 &&
|
||||
i + 22 + v.getUint16(i + 20, true) === tail.length
|
||||
) {
|
||||
end = i;
|
||||
break;
|
||||
}
|
||||
if (end < 0) throw Error("Missing ZIP directory");
|
||||
if (v.getUint16(end + 4, true) || v.getUint16(end + 6, true))
|
||||
throw Error("Multi-volume ZIPs are not supported");
|
||||
let count = v.getUint16(end + 10, true),
|
||||
size = v.getUint32(end + 12, true),
|
||||
offset = v.getUint32(end + 16, true);
|
||||
if (count === 65535 || size === 0xffffffff || offset === 0xffffffff) {
|
||||
const locatorOffset = archive.size - tail.length + end - 20,
|
||||
locator = view(await range(archive, locatorOffset, 20));
|
||||
if (
|
||||
locator.getUint32(0, true) !== 0x07064b50 ||
|
||||
locator.getUint32(4, true) !== 0 ||
|
||||
locator.getUint32(16, true) !== 1
|
||||
)
|
||||
throw Error("Invalid ZIP64 locator");
|
||||
const z = view(await range(archive, u64(locator, 8), 56));
|
||||
if (
|
||||
z.getUint32(0, true) !== 0x06064b50 ||
|
||||
z.getUint32(16, true) ||
|
||||
z.getUint32(20, true)
|
||||
)
|
||||
throw Error("Invalid ZIP64 directory");
|
||||
count = u64(z, 32);
|
||||
size = u64(z, 40);
|
||||
offset = u64(z, 48);
|
||||
}
|
||||
if (count > 100000)
|
||||
throw Error(
|
||||
"ZIP has more than 100,000 entries; use the ZIP download"
|
||||
);
|
||||
if (offset + size > archive.size || offset < 0)
|
||||
throw Error("Invalid ZIP directory bounds");
|
||||
let buffer = new Uint8Array(),
|
||||
base = -1;
|
||||
const read = async (position, length) => {
|
||||
if (position < offset || position + length > offset + size)
|
||||
throw Error("Invalid ZIP entry bounds");
|
||||
if (position < base || position + length > base + buffer.length) {
|
||||
base = position;
|
||||
buffer = await range(
|
||||
archive,
|
||||
position,
|
||||
Math.min(Math.max(CHUNK, length), offset + size - position)
|
||||
);
|
||||
}
|
||||
return buffer.subarray(position - base, position - base + length);
|
||||
};
|
||||
const entries = [];
|
||||
let cursor = offset;
|
||||
for (let i = 0; i < count; i++) {
|
||||
const h = view(await read(cursor, 46));
|
||||
if (h.getUint32(0, true) !== 0x02014b50)
|
||||
throw Error("Invalid ZIP directory entry");
|
||||
const nameLength = h.getUint16(28, true),
|
||||
extraLength = h.getUint16(30, true),
|
||||
commentLength = h.getUint16(32, true),
|
||||
flags = h.getUint16(8, true),
|
||||
method = h.getUint16(10, true),
|
||||
crc = h.getUint32(16, true),
|
||||
date = h.getUint16(14, true),
|
||||
time = h.getUint16(12, true);
|
||||
let compressed = h.getUint32(20, true),
|
||||
size = h.getUint32(24, true),
|
||||
local = h.getUint32(42, true);
|
||||
const fields = await read(
|
||||
cursor + 46,
|
||||
nameLength + extraLength + commentLength
|
||||
),
|
||||
name = decoder.decode(fields.subarray(0, nameLength)),
|
||||
extra = view(
|
||||
fields.subarray(nameLength, nameLength + extraLength)
|
||||
);
|
||||
for (let p = 0; p + 4 <= extra.byteLength; ) {
|
||||
const id = extra.getUint16(p, true),
|
||||
length = extra.getUint16(p + 2, true);
|
||||
if (p + 4 + length > extra.byteLength)
|
||||
throw Error("Invalid ZIP extra field");
|
||||
if (id === 1) {
|
||||
let q = p + 4;
|
||||
const next = () => {
|
||||
if (q + 8 > p + 4 + length)
|
||||
throw Error("Invalid ZIP64 entry");
|
||||
const n = u64(extra, q);
|
||||
q += 8;
|
||||
return n;
|
||||
};
|
||||
if (size === 0xffffffff) size = next();
|
||||
if (compressed === 0xffffffff) compressed = next();
|
||||
if (local === 0xffffffff) local = next();
|
||||
}
|
||||
p += 4 + length;
|
||||
}
|
||||
const path = safePath(name);
|
||||
if (path)
|
||||
entries.push({
|
||||
path,
|
||||
directory: name.endsWith("/"),
|
||||
size,
|
||||
compressed,
|
||||
local,
|
||||
flags,
|
||||
method,
|
||||
crc,
|
||||
archive,
|
||||
date: date
|
||||
? new Date(
|
||||
1980 + (date >> 9),
|
||||
((date >> 5) & 15) - 1,
|
||||
date & 31,
|
||||
time >> 11,
|
||||
(time >> 5) & 63,
|
||||
(time & 31) * 2
|
||||
)
|
||||
: null,
|
||||
});
|
||||
cursor += 46 + nameLength + extraLength + commentLength;
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
const crcTable = Uint32Array.from({ length: 256 }, (_, n) => {
|
||||
for (let i = 0; i < 8; i++)
|
||||
n = n & 1 ? 0xedb88320 ^ (n >>> 1) : n >>> 1;
|
||||
return n >>> 0;
|
||||
});
|
||||
async function entryStream(file, signal) {
|
||||
if (file.flags & 1)
|
||||
throw Error("Encrypted ZIP entries cannot be previewed");
|
||||
if (![0, 8].includes(file.method))
|
||||
throw Error("This ZIP compression method cannot be previewed");
|
||||
const h = view(await range(file.archive, file.local, 30, signal));
|
||||
if (h.getUint32(0, true) !== 0x04034b50)
|
||||
throw Error("Invalid ZIP file header");
|
||||
const start =
|
||||
file.local + 30 + h.getUint16(26, true) + h.getUint16(28, true);
|
||||
let stream = file.compressed
|
||||
? (
|
||||
await rangeResponse(
|
||||
file.archive,
|
||||
start,
|
||||
file.compressed,
|
||||
signal
|
||||
)
|
||||
).body
|
||||
: new Blob([]).stream();
|
||||
if (file.method === 8)
|
||||
stream = stream.pipeThrough(new DecompressionStream("deflate-raw"));
|
||||
let size = 0,
|
||||
crc = 0xffffffff;
|
||||
return stream.pipeThrough(
|
||||
new TransformStream({
|
||||
transform(chunk, controller) {
|
||||
size += chunk.length;
|
||||
if (size > file.size)
|
||||
throw Error("ZIP entry exceeds its declared size");
|
||||
for (const byte of chunk)
|
||||
crc = crcTable[(crc ^ byte) & 255] ^ (crc >>> 8);
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
flush() {
|
||||
if (size !== file.size || (crc ^ 0xffffffff) >>> 0 !== file.crc)
|
||||
throw Error("ZIP file size or checksum mismatch");
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
let files = [],
|
||||
prefix = "",
|
||||
selected = null,
|
||||
controller = null,
|
||||
blobURL = null;
|
||||
const clearPreview = () => {
|
||||
controller?.abort();
|
||||
if (blobURL) URL.revokeObjectURL(blobURL);
|
||||
blobURL = null;
|
||||
byId("content").replaceChildren();
|
||||
};
|
||||
const report = (error) => {
|
||||
if (error.name !== "AbortError")
|
||||
byId("error").textContent = error.message;
|
||||
};
|
||||
function browse(next) {
|
||||
prefix = next;
|
||||
const filter = byId("search").value.toLowerCase(),
|
||||
crumbs = byId("breadcrumbs");
|
||||
crumbs.replaceChildren();
|
||||
const crumb = (label, path) => {
|
||||
const b = el("button", label);
|
||||
b.onclick = () => {
|
||||
byId("search").value = "";
|
||||
browse(path);
|
||||
};
|
||||
crumbs.append(b);
|
||||
};
|
||||
crumb("Files", "");
|
||||
let part = "";
|
||||
for (const segment of prefix.split("/").filter(Boolean)) {
|
||||
part += segment + "/";
|
||||
crumbs.append(el("span", "/"));
|
||||
crumb(segment, part);
|
||||
}
|
||||
const items = new Map();
|
||||
for (const f of files) {
|
||||
if (!f.path.startsWith(prefix)) continue;
|
||||
const tail = f.path.slice(prefix.length),
|
||||
name = tail.split("/")[0];
|
||||
if (!name || !name.toLowerCase().includes(filter)) continue;
|
||||
const directory = f.directory || tail.includes("/");
|
||||
if (!items.has(name) || directory)
|
||||
items.set(
|
||||
name,
|
||||
directory ? { path: prefix + name, directory: true } : f
|
||||
);
|
||||
}
|
||||
const table = byId("entries");
|
||||
table.replaceChildren();
|
||||
for (const file of [...items.values()].sort(
|
||||
(a, b) =>
|
||||
Number(b.directory) - Number(a.directory) ||
|
||||
a.path.localeCompare(b.path)
|
||||
)) {
|
||||
const row = el("tr");
|
||||
if (file === selected) row.className = "selected";
|
||||
const name = el("td", null, "name"),
|
||||
button = el("button", null, "entry");
|
||||
button.append(
|
||||
el("span", file.directory ? "📁" : "📄"),
|
||||
el("span", file.path.slice(prefix.length))
|
||||
);
|
||||
button.onclick = () =>
|
||||
file.directory ? browse(file.path + "/") : openFile(file);
|
||||
name.append(button);
|
||||
row.append(
|
||||
name,
|
||||
el("td", file.directory ? "—" : bytes(file.size), "size"),
|
||||
el("td", file.date?.toLocaleDateString() || "", "date")
|
||||
);
|
||||
table.append(row);
|
||||
}
|
||||
byId("count").textContent = items.size + " items";
|
||||
if (!items.size) {
|
||||
const row = el("tr"),
|
||||
cell = el("td", "No files", "empty");
|
||||
cell.colSpan = 3;
|
||||
row.append(cell);
|
||||
table.append(row);
|
||||
}
|
||||
}
|
||||
const mime = (path) =>
|
||||
({
|
||||
png: "image/png",
|
||||
jpg: "image/jpeg",
|
||||
jpeg: "image/jpeg",
|
||||
gif: "image/gif",
|
||||
webp: "image/webp",
|
||||
avif: "image/avif",
|
||||
pdf: "application/pdf",
|
||||
mp3: "audio/mpeg",
|
||||
ogg: "audio/ogg",
|
||||
wav: "audio/wav",
|
||||
mp4: "video/mp4",
|
||||
webm: "video/webm",
|
||||
}[path.split(".").pop().toLowerCase()] || "application/octet-stream");
|
||||
async function openFile(file) {
|
||||
clearPreview();
|
||||
selected = file;
|
||||
controller = new AbortController();
|
||||
const signal = controller.signal;
|
||||
byId("error").textContent = "";
|
||||
byId("preview").hidden = false;
|
||||
byId("filename").textContent = file.path.split("/").pop();
|
||||
browse(prefix);
|
||||
const content = byId("content");
|
||||
content.append(el("span", "Loading…", "empty"));
|
||||
if (file.size > MAX_PREVIEW) {
|
||||
content.replaceChildren(
|
||||
el("span", "File is too large to preview. Use Download.", "empty")
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const stream = file.archive
|
||||
? await entryStream(file, signal)
|
||||
: (await fetch(file.url, { signal })).body;
|
||||
const blob = await new Response(stream).blob();
|
||||
if (signal.aborted) return;
|
||||
const type = mime(file.path);
|
||||
content.replaceChildren();
|
||||
if (type !== "application/octet-stream") {
|
||||
blobURL = URL.createObjectURL(new Blob([blob], { type }));
|
||||
let node;
|
||||
if (type.startsWith("image/")) {
|
||||
node = el("img");
|
||||
node.alt = file.path;
|
||||
} else if (type === "application/pdf") {
|
||||
node = el("iframe");
|
||||
node.title = file.path;
|
||||
node.setAttribute("sandbox", "");
|
||||
} else {
|
||||
node = el(type.startsWith("audio/") ? "audio" : "video");
|
||||
node.controls = true;
|
||||
}
|
||||
node.src = blobURL;
|
||||
content.append(node);
|
||||
} else if (
|
||||
/\.(txt|md|csv|tsv|json|xml|html?|css|js|py|log|ya?ml|svg|rtf)$/i.test(
|
||||
file.path
|
||||
)
|
||||
)
|
||||
content.append(el("pre", await blob.text()));
|
||||
else
|
||||
content.append(
|
||||
el(
|
||||
"span",
|
||||
"Preview unavailable for this file type. Use Download.",
|
||||
"empty"
|
||||
)
|
||||
);
|
||||
} catch (error) {
|
||||
if (!signal.aborted) {
|
||||
content.replaceChildren();
|
||||
report(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
byId("search").oninput = () => browse(prefix);
|
||||
byId("close").onclick = () => {
|
||||
clearPreview();
|
||||
selected = null;
|
||||
byId("preview").hidden = true;
|
||||
browse(prefix);
|
||||
};
|
||||
byId("download").onclick = async () => {
|
||||
if (!selected) return;
|
||||
const file = selected;
|
||||
byId("error").textContent = "";
|
||||
if (!file.archive) {
|
||||
const a = el("a");
|
||||
a.href = file.url;
|
||||
a.download = file.path;
|
||||
a.click();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// Stream large entries to disk where supported; never buffer an entire ZIP.
|
||||
if ("showSaveFilePicker" in window) {
|
||||
const handle = await window.showSaveFilePicker({
|
||||
suggestedName: file.path.split("/").pop(),
|
||||
});
|
||||
const writable = await handle.createWritable();
|
||||
try {
|
||||
await (await entryStream(file)).pipeTo(writable);
|
||||
} catch (e) {
|
||||
await writable.abort().catch(() => {});
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
if (file.size > MAX_PREVIEW)
|
||||
throw Error(
|
||||
"Large individual downloads require a browser with streaming file saves. Use ZIP to download the archive."
|
||||
);
|
||||
const blob = await new Response(await entryStream(file)).blob(),
|
||||
url = URL.createObjectURL(blob),
|
||||
a = el("a");
|
||||
a.href = url;
|
||||
a.download = file.path.split("/").pop();
|
||||
a.click();
|
||||
setTimeout(() => URL.revokeObjectURL(url), 60000);
|
||||
}
|
||||
} catch (error) {
|
||||
report(error);
|
||||
}
|
||||
};
|
||||
const download = el("a", "Download ZIP", "action");
|
||||
download.href = raw.href;
|
||||
download.download = "";
|
||||
byId("actions").insertBefore(download, byId("files"));
|
||||
document.title = byId("title").textContent;
|
||||
indexZip({ url: raw.href, size: Number(document.body.dataset.size) })
|
||||
.then((entries) => {
|
||||
files = entries;
|
||||
browse("");
|
||||
})
|
||||
.catch(report);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -14,6 +14,7 @@ import textwrap
|
||||
import time
|
||||
import shutil
|
||||
import ctypes
|
||||
from urllib.parse import urlsplit
|
||||
from datetime import timedelta
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
@@ -22,6 +23,13 @@ from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from collections.abc import Callable
|
||||
|
||||
from abx_plugins.plugins.opencode.archivebox.conftest import (
|
||||
opencode_archive_config as opencode_archive_config,
|
||||
installed_opencode as installed_opencode,
|
||||
live_opencode as live_opencode,
|
||||
agent_server as agent_server,
|
||||
)
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
import requests
|
||||
@@ -32,6 +40,7 @@ PYTEST_BASETEMP_ROOT = (REPO_ROOT / "tests" / "out").resolve()
|
||||
SESSION_DATA_DIR = Path(
|
||||
os.environ.get("ARCHIVEBOX_PYTEST_SESSION_DATA_DIR") or tempfile.mkdtemp(prefix="archivebox-pytest-session-"),
|
||||
).resolve()
|
||||
PORT_ALLOCATION_DIR = Path(os.environ.setdefault("ARCHIVEBOX_PYTEST_PORTS_DIR", str(SESSION_DATA_DIR / "ports")))
|
||||
if worker_id := os.environ.get("PYTEST_XDIST_WORKER"):
|
||||
# Workers inherit the coordinator's environment. Give each one its own
|
||||
# collection and cleanup root so a finishing worker cannot delete another's.
|
||||
@@ -330,6 +339,18 @@ def pytest_configure():
|
||||
if not apps.ready:
|
||||
django.setup()
|
||||
|
||||
if os.environ.get("PYTEST_XDIST_WORKER"):
|
||||
from django.conf import settings
|
||||
from archivebox.config import django as django_config
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
# pytest-django may import startup config before this conftest isolates
|
||||
# the worker's paths. Refresh that real startup config once so HTTP
|
||||
# requests resolve the same installed binaries as worker subprocesses;
|
||||
# otherwise a service restart searches the coordinator's empty lib dir.
|
||||
django_config.CONFIG = get_config(include_machine=False)
|
||||
settings.CONFIG = django_config.CONFIG
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def django_db_modify_db_settings(
|
||||
@@ -444,7 +465,7 @@ def hermetic_lib_dir(tmp_path):
|
||||
@pytest.fixture
|
||||
def cached_abxpkg_lib_dir():
|
||||
"""Reuse one real abxpkg installation cache for the current pytest session."""
|
||||
lib_dir = SESSION_DATA_DIR / "lib"
|
||||
lib_dir = Path(os.environ["ABXPKG_LIB_DIR"]) if os.environ.get("ABXPKG_LIB_DIR") else SESSION_DATA_DIR / "lib"
|
||||
lib_dir.mkdir(parents=True, exist_ok=True)
|
||||
original_lib_dir = os.environ.get("ABXPKG_LIB_DIR")
|
||||
os.environ["ABXPKG_LIB_DIR"] = str(lib_dir)
|
||||
@@ -1083,6 +1104,8 @@ def start_archivebox_server(
|
||||
else:
|
||||
output = log_path.read_text(encoding="utf-8", errors="replace") if log_path else ""
|
||||
raise AssertionError(f"ArchiveBox server did not open port {port} within 30 seconds:\n{output}")
|
||||
response = get_http_response(port, host=f"127.0.0.1:{port}", path="/health/", timeout=5)
|
||||
assert response.status_code == 200 and response.headers.get("X-ArchiveBox-Health") == "OK", response.text
|
||||
return proc
|
||||
|
||||
|
||||
@@ -1301,6 +1324,43 @@ def api_auth_headers(api_token: str, *, django_client: bool = False, port: int |
|
||||
}
|
||||
|
||||
|
||||
class LocalhostHTTPAdapter(requests.adapters.HTTPAdapter):
|
||||
"""Connect real HTTP requests to loopback while preserving URL authorities."""
|
||||
|
||||
def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None):
|
||||
url = urlsplit(request.url)
|
||||
if url.scheme != "http" or not (url.hostname == "localhost" or (url.hostname or "").endswith(".localhost")):
|
||||
raise ValueError(f"Expected an HTTP localhost test URL, got {request.url}")
|
||||
return self.poolmanager.connection_from_host("127.0.0.1", port=url.port, scheme="http")
|
||||
|
||||
def send(self, request, **kwargs):
|
||||
# Send Host only on the wire. Requests copies prepared headers on
|
||||
# redirects and uses Host for cookie matching; leaving it attached
|
||||
# would select the previous origin's cookies after a cross-host redirect.
|
||||
original_host = request.headers.get("Host")
|
||||
request.headers["Host"] = urlsplit(request.url).netloc
|
||||
try:
|
||||
return super().send(request, **kwargs)
|
||||
finally:
|
||||
if original_host is None:
|
||||
request.headers.pop("Host", None)
|
||||
else:
|
||||
request.headers["Host"] = original_host
|
||||
|
||||
|
||||
def localhost_session() -> requests.Session:
|
||||
# Browsers resolve *.localhost themselves; Linux libc in stock containers
|
||||
# often does not. This is curl --connect-to for real server tests: only the
|
||||
# socket destination changes. Keep the original URL so Requests exercises
|
||||
# real redirect and cookie-domain rules, including dynamic snap-* hosts.
|
||||
# Never replace URLs with 127.0.0.1 or stub responses: that hides isolation
|
||||
# bugs. This also avoids global /etc/hosts edits and runner-specific DNS.
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
session.mount("http://", LocalhostHTTPAdapter())
|
||||
return session
|
||||
|
||||
|
||||
def wait_for_live_api(port: int, *, path: str = "/api/v1/docs"):
|
||||
return get_http_response(port, host=f"api.archivebox.localhost:{port}", path=path)
|
||||
|
||||
@@ -1346,9 +1406,20 @@ def init_archive(cwd: Path) -> None:
|
||||
|
||||
|
||||
def get_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return sock.getsockname()[1]
|
||||
# Binding port zero only proves availability now. Another pytest worker can
|
||||
# receive that same port before either service starts. Claim it atomically
|
||||
# for this session while the socket is still bound, across all workers and
|
||||
# helper subprocesses; the session cleanup removes these small markers.
|
||||
PORT_ALLOCATION_DIR.mkdir(parents=True, exist_ok=True)
|
||||
while True:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
try:
|
||||
(PORT_ALLOCATION_DIR / str(port)).touch(exist_ok=False)
|
||||
except FileExistsError:
|
||||
continue
|
||||
return port
|
||||
|
||||
|
||||
def stop_server(cwd: Path) -> None:
|
||||
@@ -1944,3 +2015,60 @@ def create_test_snapshot_json(url: str | None = None, **kwargs) -> dict[str, Any
|
||||
"status": kwargs.get("status", "queued"),
|
||||
**{k: v for k, v in kwargs.items() if k not in ("tags_str", "status")},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def browser_runtime(cached_abxpkg_lib_dir):
|
||||
from abx_plugins import get_plugins_dir
|
||||
from abx_plugins.plugins.base.utils import get_config
|
||||
|
||||
# UI tests need isolated collections, but share the browser installation.
|
||||
# Resolve it through the same abxpkg schema/cache as capture hooks rather
|
||||
# than downloading Playwright's separate Chromium in CI. This keeps local
|
||||
# and CI runs on the supported install path without per-test cold installs.
|
||||
shared_lib = cached_abxpkg_lib_dir
|
||||
env = cli_env(
|
||||
ABXPKG_LIB_DIR=str(shared_lib),
|
||||
CHROME_HEADLESS="True",
|
||||
CHROME_SANDBOX="False",
|
||||
CHROME_ISOLATION="snapshot",
|
||||
)
|
||||
env.pop("CHROME_BINARY", None)
|
||||
resolved_env = resolve_abxpkg_chrome_env(shared_lib, env)
|
||||
chrome_config = get_config(
|
||||
Path(get_plugins_dir()) / "chrome" / "config.json",
|
||||
environ={**env, **resolved_env},
|
||||
hydrate_binaries=False,
|
||||
)
|
||||
|
||||
return {
|
||||
"lib_dir": shared_lib,
|
||||
"node_modules_dir": Path(resolved_env["NODE_MODULES_DIR"]),
|
||||
"node_path": resolved_env["NODE_PATH"],
|
||||
"node_binary": Path(resolved_env["NODE_BINARY"]),
|
||||
"chrome_binary": Path(resolved_env["CHROME_BINARY"]),
|
||||
"chrome_args": [*chrome_config.CHROME_ARGS, *chrome_config.CHROME_ARGS_EXTRA],
|
||||
}
|
||||
|
||||
|
||||
def _reset_runtime_config() -> None:
|
||||
from archivebox.config import common
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
for value in vars(common).values():
|
||||
cache_clear = getattr(value, "cache_clear", None)
|
||||
if cache_clear is not None:
|
||||
cache_clear()
|
||||
Machine.current()
|
||||
|
||||
|
||||
def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
|
||||
os.chdir(data_dir)
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", *values],
|
||||
cwd=data_dir,
|
||||
env=env,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
_reset_runtime_config()
|
||||
@@ -0,0 +1,13 @@
|
||||
# Responsive-image replay regression
|
||||
|
||||
`google-drive.wacz` contains two unmodified, gzip-compressed WARC response
|
||||
records from a real ArchiveWeb.page capture on 2026-10-04:
|
||||
|
||||
- https://drive.google.com/drive/folders/1KpLl_1tcK0eeehzN980zbG-3M2nhbVks
|
||||
- https://www.gstatic.com/images/branding/productlogos/drive_2026/v1/web-48dp/logo_drive_2026_color_1x_web_48dp.png
|
||||
|
||||
The public folder's HTML declares both 1x and 2x logo URLs, but the recording
|
||||
only contains the 1x image. These records were copied byte-for-byte from the
|
||||
capture's `archive/data.warc.gz` using its CDX offsets; the subset CDX offsets and datapackage hashes were recomputed. No HTTP
|
||||
responses were synthesized or edited. Other resources were omitted to keep the fixture small.
|
||||
The test replays this actual archive at DPR 1 and 2, entirely from local storage.
|
||||
Binary file not shown.
@@ -15,11 +15,19 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
def test_browser_metadata_respects_only_new_and_reports_capture_owner(client, api_headers):
|
||||
url = "https://example.com/browser-deduplication"
|
||||
for name in ("Original", "Replacement"):
|
||||
persona = client.post(
|
||||
"/api/v1/personas/sync",
|
||||
data={"extension_persona_id": f"dedupe-{name}", "name": name},
|
||||
content_type="application/json",
|
||||
**api_headers,
|
||||
)
|
||||
assert persona.status_code == 200, persona.content
|
||||
|
||||
def submit(only_new):
|
||||
def submit(only_new, persona):
|
||||
queued = client.post(
|
||||
"/api/v1/cli/add",
|
||||
data={"urls": [url], "only_new": only_new},
|
||||
data={"urls": [url], "only_new": only_new, "persona": persona},
|
||||
content_type="application/json",
|
||||
**api_headers,
|
||||
)
|
||||
@@ -34,15 +42,17 @@ def test_browser_metadata_respects_only_new_and_reports_capture_owner(client, ap
|
||||
assert metadata.status_code == 200, metadata.content
|
||||
return crawl_id, metadata.json()
|
||||
|
||||
first_crawl, first = submit(True)
|
||||
second_crawl, reused = submit(True)
|
||||
first_crawl, first = submit(True, "Original")
|
||||
second_crawl, reused = submit(True, "Replacement")
|
||||
assert second_crawl != first_crawl
|
||||
assert reused["id"] == first["id"]
|
||||
assert reused["crawl_id"] == first_crawl
|
||||
assert reused["persona"] == "Original"
|
||||
assert Snapshot.objects.filter(url=url).count() == 1
|
||||
forced_crawl, forced = submit(False)
|
||||
forced_crawl, forced = submit(False, "Replacement")
|
||||
assert forced["id"] != first["id"]
|
||||
assert forced["crawl_id"] == forced_crawl
|
||||
assert forced["persona"] == "Replacement"
|
||||
assert Snapshot.objects.filter(url=url).count() == 2
|
||||
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ def _run_shipped_snapshot_hook(
|
||||
|
||||
from abx_dl.services.process_service import ProcessService as HookProcessService
|
||||
from abx_dl.services.archive_result_service import ArchiveResultService as HookArchiveResultService
|
||||
from abx_plugins.plugins.base.utils import _collect_required_schema_paths, get_hydrated_required_binaries
|
||||
from abx_plugins.plugins.base.utils import _collect_required_schema_paths
|
||||
from abxpkg import prepare_script_exec_plan
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.machine.models import Process
|
||||
@@ -66,15 +66,9 @@ def _run_shipped_snapshot_hook(
|
||||
hook_path = matching_hooks[0].path
|
||||
projected_hook_name = event_hook_name or hook_name
|
||||
hook_config = hook_path.parent / "config.json"
|
||||
for required_binary in get_hydrated_required_binaries(
|
||||
hook_config,
|
||||
environ={**os.environ, "ABXPKG_LIB_DIR": str(lib_dir)},
|
||||
):
|
||||
install_real_binary(
|
||||
required_binary["name"],
|
||||
binproviders=required_binary["binproviders"],
|
||||
overrides=required_binary.get("overrides"),
|
||||
)
|
||||
# This helper runs standalone hooks. Resolve their dependencies through the
|
||||
# public abxpkg CLI below; creating a new ArchiveBox Binary row on every
|
||||
# invocation makes a second run collide with the already-installed identity.
|
||||
binary_env = resolve_abxpkg_binary_env(lib_dir, deps_from=_collect_required_schema_paths(hook_config))
|
||||
hook_env = {
|
||||
**binary_env,
|
||||
|
||||
@@ -198,21 +198,26 @@ def test_binary_request_installs_env_binary_and_recovers_stale_cache(initialized
|
||||
assert Path(relibbed.abspath).resolve() == Path(host_binary).resolve()
|
||||
|
||||
|
||||
def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive, tmp_path):
|
||||
name = "http"
|
||||
def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_resolve(initialized_archive):
|
||||
name = "httpie"
|
||||
provider_bin_dir = initialized_archive / "lib" / "pip" / "venv" / "bin"
|
||||
runtime_env = _runtime_env(initialized_archive)
|
||||
|
||||
_cmd_result = _run_real_binary_lifecycle(initialized_archive, name=name, binproviders="env", env=runtime_env)
|
||||
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert returncode != 0, stdout + stderr
|
||||
assert "Binary http installation failed" in stderr
|
||||
failed = run_archivebox_cmd(
|
||||
["install", "--binproviders=env", name],
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=runtime_env,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert failed.returncode != 0, failed.stdout + failed.stderr
|
||||
assert f"Binary {name} installation failed" in failed.stderr
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
queued = Binary.objects.get(name=name)
|
||||
queued_id = str(queued.id)
|
||||
failed_process = Process.objects.filter(process_type=Process.TypeChoices.BINARY).latest("created_at")
|
||||
failed_process = Process.objects.filter(binary_id=queued_id).latest("created_at")
|
||||
machine_config = Machine.objects.get(pk=queued.machine_id).config or {}
|
||||
|
||||
assert queued.status == Binary.StatusChoices.QUEUED
|
||||
@@ -223,28 +228,42 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
|
||||
assert f"{name.upper().replace('-', '_')}_BINARY" not in machine_config
|
||||
assert not (provider_bin_dir / name).exists()
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
queued = Binary.objects.get(pk=queued_id)
|
||||
queued.binproviders = "pip"
|
||||
queued.overrides = {"pip": {"install_args": ["httpie>=3.2.4"]}}
|
||||
queued.save(update_fields=["binproviders", "overrides", "modified_at"])
|
||||
recovered_runtime_env = _runtime_env(initialized_archive)
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run", f"--binary-id={queued_id}"],
|
||||
recovered_install = run_archivebox_cmd(
|
||||
["install", "--binproviders=pip", name],
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=recovered_runtime_env,
|
||||
env=runtime_env,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
recover_stdout, recover_stderr, recover_code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert recover_code == 0, recover_stdout + recover_stderr
|
||||
assert recovered_install.returncode == 0, recovered_install.stdout + recovered_install.stderr
|
||||
repeated_install = run_archivebox_cmd(
|
||||
["install", "--binproviders=pip", name],
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=runtime_env,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert repeated_install.returncode == 0, repeated_install.stdout + repeated_install.stderr
|
||||
# Changing the provider policy runs a real installation request against the
|
||||
# now-warm cache. Its fast result must still finalize the DB projection.
|
||||
cached_install = run_archivebox_cmd(
|
||||
["install", "--binproviders=pip,env", name],
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=runtime_env,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert cached_install.returncode == 0, cached_install.stdout + cached_install.stderr
|
||||
with use_archivebox_db(initialized_archive):
|
||||
recovered = Binary.objects.get(pk=queued_id)
|
||||
# Providers may resolve their own dependencies during installation. Those
|
||||
# Process rows are not attempts to install this binary; track its exact
|
||||
# identity instead of assuming the last two global rows belong to it.
|
||||
process_exit_codes = list(
|
||||
Process.objects.filter(process_type=Process.TypeChoices.BINARY).order_by("created_at").values_list("exit_code", flat=True),
|
||||
Process.objects.filter(binary_id=queued_id).order_by("created_at").values_list("exit_code", flat=True),
|
||||
)
|
||||
|
||||
assert recovered.status == Binary.StatusChoices.INSTALLED
|
||||
@@ -253,4 +272,4 @@ def test_missing_binary_request_stays_queued_then_recovers_when_provider_can_res
|
||||
assert Path(recovered.abspath) == provider_bin_dir / name
|
||||
assert recovered.binprovider == "pip"
|
||||
assert Path(recovered.abspath).is_file()
|
||||
assert process_exit_codes[-2:] == [1, 0]
|
||||
assert process_exit_codes == [1, 0, 0]
|
||||
@@ -8,37 +8,20 @@ from archivebox.tests.conftest import run_archivebox_cmd
|
||||
from archivebox.config.configset import read_ini_config
|
||||
|
||||
|
||||
def test_config_displays_all_config(initialized_archive):
|
||||
"""Test that config without args displays all configuration."""
|
||||
result = run_archivebox_cmd(["config"])
|
||||
|
||||
assert result.returncode == 0
|
||||
output = result.stdout
|
||||
# Should show config sections
|
||||
assert len(output) > 100
|
||||
# Should show at least some standard config keys
|
||||
assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output
|
||||
|
||||
|
||||
def test_config_shows_derived_collection_paths_but_not_runtime_dirs(initialized_archive):
|
||||
"""The CLI should expose collection paths, not per-crawl/per-snapshot runtime dirs."""
|
||||
run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
|
||||
|
||||
def test_config_read_get_and_search_cover_core_plugin_alias_and_path_options(initialized_archive):
|
||||
"""Exercise every read-only CLI branch against one unchanged collection."""
|
||||
result = run_archivebox_cmd(["config"], cwd=initialized_archive)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
unwrapped_output = result.stdout.replace("\n", "")
|
||||
assert "DATA_DIR" in result.stdout
|
||||
assert result.stdout.count("\nDATA_DIR =") == 1
|
||||
assert str(initialized_archive) in unwrapped_output
|
||||
assert "PERSONAS_DIR" in result.stdout
|
||||
assert result.stdout.count("\nPERSONAS_DIR =") == 1
|
||||
assert "SNAP_DIR" not in result.stdout
|
||||
assert "CRAWL_DIR" not in result.stdout
|
||||
|
||||
|
||||
def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive):
|
||||
run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
|
||||
output = result.stdout
|
||||
assert len(output) > 100
|
||||
assert "TIMEOUT" in output or "OUTPUT_PERMISSIONS" in output
|
||||
assert "DATA_DIR" in output
|
||||
assert output.count("\nDATA_DIR =") == 1
|
||||
assert str(initialized_archive) in output.replace("\n", "")
|
||||
assert "PERSONAS_DIR" in output
|
||||
assert output.count("\nPERSONAS_DIR =") == 1
|
||||
assert "SNAP_DIR" not in output
|
||||
assert "CRAWL_DIR" not in output
|
||||
|
||||
data_dir = run_archivebox_cmd(["config", "--get", "DATA_DIR"], cwd=initialized_archive)
|
||||
snap_dir = run_archivebox_cmd(["config", "--get", "SNAP_DIR"], cwd=initialized_archive)
|
||||
@@ -49,10 +32,21 @@ def test_config_get_derived_path_but_rejects_runtime_dir(initialized_archive):
|
||||
assert snap_dir.returncode != 0
|
||||
assert "SNAP_DIR =" not in snap_dir.stdout
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--get", "TIMEOUT"],
|
||||
)
|
||||
|
||||
def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive):
|
||||
run_archivebox_cmd(["init"], cwd=initialized_archive, check=True)
|
||||
assert result.returncode == 0
|
||||
assert "TIMEOUT" in result.stdout
|
||||
|
||||
for query, expected in (("TIMEOUT", "TIMEOUT"), ("wget", "WGET_BINARY"), ("URL_BLACK", "URL_DENYLIST")):
|
||||
result = run_archivebox_cmd(["config", "--search", query])
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert expected in result.stdout, query
|
||||
|
||||
|
||||
def test_config_rejects_readonly_runtime_unknown_and_malformed_keys(initialized_archive):
|
||||
"""Each rejected input must fail through its own CLI validation branch."""
|
||||
data_dir = run_archivebox_cmd(
|
||||
["config", "--set", f"DATA_DIR={initialized_archive / 'other'}"],
|
||||
cwd=initialized_archive,
|
||||
@@ -68,87 +62,12 @@ def test_config_set_rejects_readonly_and_runtime_dirs(initialized_archive):
|
||||
assert "DATA_DIR" not in content
|
||||
assert "CRAWL_DIR" not in content
|
||||
|
||||
|
||||
def test_config_get_specific_key(initialized_archive):
|
||||
"""Test that config --get KEY retrieves specific value."""
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--get", "TIMEOUT"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "TIMEOUT" in result.stdout
|
||||
|
||||
|
||||
def test_config_set_writes_to_file(initialized_archive):
|
||||
"""Test that config --set KEY=VALUE writes to ArchiveBox.conf."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=120"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "TIMEOUT=120" in result.stdout
|
||||
|
||||
# Verify config file was updated
|
||||
config_file = initialized_archive / "ArchiveBox.conf"
|
||||
assert config_file.exists()
|
||||
|
||||
content = config_file.read_text()
|
||||
assert "TIMEOUT" in content or "120" in content
|
||||
|
||||
|
||||
def test_config_set_and_get_roundtrip(initialized_archive):
|
||||
"""Test that set value can be retrieved with get."""
|
||||
|
||||
# Set a unique value
|
||||
run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=987"],
|
||||
)
|
||||
|
||||
updated = run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=654"],
|
||||
)
|
||||
|
||||
assert updated.returncode == 0
|
||||
assert "TIMEOUT=654" in updated.stdout
|
||||
|
||||
# Get the value back
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--get", "TIMEOUT"],
|
||||
)
|
||||
|
||||
assert "654" in result.stdout
|
||||
|
||||
|
||||
def test_config_set_multiple_values(initialized_archive):
|
||||
"""Test setting multiple config values at once."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
# Verify both were written
|
||||
config_file = initialized_archive / "ArchiveBox.conf"
|
||||
content = config_file.read_text()
|
||||
assert "111" in content
|
||||
assert "222" in content
|
||||
|
||||
|
||||
def test_config_set_invalid_key_fails(initialized_archive):
|
||||
"""Test that setting invalid config key fails."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", "TOTALLY_INVALID_KEY_XYZ=value"],
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
def test_config_set_requires_equals_sign(initialized_archive):
|
||||
"""Test that set requires KEY=VALUE format."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT"],
|
||||
)
|
||||
@@ -156,93 +75,46 @@ def test_config_set_requires_equals_sign(initialized_archive):
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
def test_config_search_finds_keys(initialized_archive):
|
||||
"""Test that config --search finds matching keys."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--search", "TIMEOUT"],
|
||||
)
|
||||
|
||||
# Should find timeout-related config
|
||||
assert "TIMEOUT" in result.stdout
|
||||
|
||||
|
||||
def test_config_search_finds_plugin_options(initialized_archive):
|
||||
"""Test that config --search finds plugin keys and descriptions."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--search", "wget"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "WGET_BINARY" in result.stdout
|
||||
|
||||
|
||||
def test_config_search_finds_core_aliases(initialized_archive):
|
||||
"""Test that config --search finds core options by partial alias."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--search", "URL_BLACK"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "URL_DENYLIST" in result.stdout
|
||||
|
||||
|
||||
def test_config_preserves_existing_values(initialized_archive):
|
||||
"""Test that setting new values preserves existing ones."""
|
||||
|
||||
# Set first value
|
||||
run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=100"],
|
||||
)
|
||||
|
||||
# Set second value
|
||||
run_archivebox_cmd(
|
||||
["config", "--set", "YTDLP_TIMEOUT=200"],
|
||||
)
|
||||
|
||||
# Verify both are in config file
|
||||
def test_config_set_update_get_preserves_other_keys(initialized_archive):
|
||||
"""Exercise each write path against one real config, checking each transition."""
|
||||
result = run_archivebox_cmd(["config", "--set", "TIMEOUT=120"])
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "TIMEOUT=120" in result.stdout
|
||||
config_file = initialized_archive / "ArchiveBox.conf"
|
||||
assert config_file.is_file()
|
||||
content = config_file.read_text()
|
||||
assert "TIMEOUT" in content and "120" in content
|
||||
assert "[" in content or "=" in content
|
||||
assert read_ini_config(config_file)["TIMEOUT"] == "120"
|
||||
|
||||
result = run_archivebox_cmd(["config", "--set", "YTDLP_TIMEOUT=200"])
|
||||
assert result.returncode == 0, result.stderr
|
||||
content = config_file.read_text()
|
||||
assert "TIMEOUT" in content
|
||||
assert "YTDLP_TIMEOUT" in content
|
||||
assert read_ini_config(config_file)["TIMEOUT"] == "120"
|
||||
assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "200"
|
||||
|
||||
updated = run_archivebox_cmd(["config", "--set", "TIMEOUT=654"])
|
||||
assert updated.returncode == 0, updated.stderr
|
||||
assert "TIMEOUT=654" in updated.stdout
|
||||
result = run_archivebox_cmd(["config", "--get", "TIMEOUT"])
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "654" in result.stdout
|
||||
assert read_ini_config(config_file)["TIMEOUT"] == "654"
|
||||
assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "200"
|
||||
|
||||
|
||||
def test_config_file_is_valid_toml(initialized_archive):
|
||||
"""Test that config file remains valid TOML after set."""
|
||||
|
||||
run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=150"],
|
||||
)
|
||||
|
||||
def test_config_set_multiple_values_in_fresh_sections(initialized_archive):
|
||||
"""Keep first writes into new sections distinct from updating existing keys."""
|
||||
result = run_archivebox_cmd(["config", "--set", "TIMEOUT=111", "YTDLP_TIMEOUT=222"])
|
||||
assert result.returncode == 0, result.stderr
|
||||
config_file = initialized_archive / "ArchiveBox.conf"
|
||||
content = config_file.read_text()
|
||||
|
||||
# Basic TOML validation - should have sections and key=value pairs
|
||||
assert "[" in content or "=" in content
|
||||
|
||||
|
||||
def test_config_updates_existing_value(initialized_archive):
|
||||
"""Test that setting same key twice updates the value."""
|
||||
|
||||
# Set initial value
|
||||
run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=100"],
|
||||
)
|
||||
|
||||
# Update to new value
|
||||
run_archivebox_cmd(
|
||||
["config", "--set", "TIMEOUT=200"],
|
||||
)
|
||||
|
||||
# Get current value
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--get", "TIMEOUT"],
|
||||
)
|
||||
|
||||
# Should show updated value
|
||||
assert "200" in result.stdout
|
||||
assert "111" in content
|
||||
assert "222" in content
|
||||
assert read_ini_config(config_file)["TIMEOUT"] == "111"
|
||||
assert read_ini_config(config_file)["YTDLP_TIMEOUT"] == "222"
|
||||
|
||||
|
||||
def test_config_ignores_legacy_unknown_keys(tmp_path, initialized_archive):
|
||||
|
||||
@@ -8,10 +8,12 @@ from urllib.parse import urlsplit
|
||||
import pytest
|
||||
from abx_plugins import get_plugins_dir
|
||||
from django.test import Client
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.core.routes_util import build_snapshot_url
|
||||
from archivebox.tests.conftest import cli_env, find_snapshot_dir, parse_jsonl_output, run_archivebox_cmd
|
||||
from archivebox.tests.conftest import cli_env, create_admin_and_token, find_snapshot_dir, parse_jsonl_output, run_archivebox_cmd
|
||||
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
@@ -80,8 +82,29 @@ def test_extract_runs_on_existing_snapshots(archive_with_extractors):
|
||||
assert archiveresults["wget"].output_str == "wget/example.com/index.html"
|
||||
assert archiveresults["wget"].output_files["example.com/index.html"]["size"] == wget_path.stat().st_size
|
||||
|
||||
token = create_admin_and_token(initialized_archive)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
response = Client().get(
|
||||
f"/api/v1/core/snapshot/{snapshot_id}",
|
||||
HTTP_HOST="api.archivebox.localhost",
|
||||
HTTP_X_ARCHIVEBOX_API_KEY=token,
|
||||
)
|
||||
assert response.status_code == 200, response.content
|
||||
returned = {row["plugin"]: row for row in response.json()["archiveresults"]}
|
||||
for plugin, result in archiveresults.items():
|
||||
assert returned[plugin]["cmd"] == result.cmd
|
||||
assert returned[plugin]["cmd_version"] == result.cmd_version
|
||||
assert returned[plugin]["pwd"] == result.pwd
|
||||
# Each real hook owns a Process and Binary. Metadata reads must fetch
|
||||
# them with the result rows, not run two extra queries per plugin.
|
||||
related_lookups = [
|
||||
query["sql"] for query in queries if 'FROM "machine_process"' in query["sql"] or 'FROM "machine_binary"' in query["sql"]
|
||||
]
|
||||
assert related_lookups == []
|
||||
|
||||
def test_wget_literal_percent_output_url_serves_captured_file(initialized_archive, snapshot, live_server):
|
||||
|
||||
def test_wget_literal_percent_output_url_serves_captured_file(initialized_archive, snapshot, live_server, browser_runtime):
|
||||
"""Replay the real HedgeDoc link-interstitial file saved by wget."""
|
||||
import shutil
|
||||
from urllib.parse import quote
|
||||
@@ -155,7 +178,10 @@ def test_wget_literal_percent_output_url_serves_captured_file(initialized_archiv
|
||||
)
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
page.goto(f"http://web.archivebox.localhost:{port}{ui_snapshot.get_absolute_url()}/index.html", wait_until="domcontentloaded")
|
||||
expected_path = quote(str(output_path), safe="/@=")
|
||||
|
||||
@@ -95,110 +95,33 @@ def test_cli_refuses_source_root_without_side_effects_and_allows_separate_data_d
|
||||
assert (data_dir / "logs").is_dir()
|
||||
|
||||
|
||||
def test_init_creates_database_file(tmp_path):
|
||||
"""Test that init creates index.sqlite3 database file."""
|
||||
def test_init_creates_collection_schema_directories_config_and_permissions(tmp_path):
|
||||
"""Inspect all outputs of one real fresh init; upgrade cases stay separate."""
|
||||
result = run_archivebox_cmd(["init"])
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
assert result.returncode == 0
|
||||
db_path = tmp_path / "index.sqlite3"
|
||||
assert db_path.exists()
|
||||
assert db_path.is_file()
|
||||
|
||||
|
||||
def test_init_creates_archive_directory(tmp_path):
|
||||
"""Test that init creates archive directory."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
archive_dir = tmp_path / "archive"
|
||||
assert archive_dir.exists()
|
||||
assert archive_dir.is_dir()
|
||||
|
||||
|
||||
def test_init_uses_cwd_archive_and_users_dirs(tmp_path):
|
||||
"""Test that init creates archive/users storage roots under cwd."""
|
||||
|
||||
result = run_archivebox_cmd(["init"])
|
||||
|
||||
assert result.returncode == 0
|
||||
assert (tmp_path / "archive").is_dir()
|
||||
assert (tmp_path / "archive" / "users").is_dir()
|
||||
|
||||
|
||||
def test_init_creates_sources_directory(tmp_path):
|
||||
"""Test that init creates sources directory."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
sources_dir = tmp_path / "sources"
|
||||
assert sources_dir.exists()
|
||||
assert sources_dir.is_dir()
|
||||
|
||||
|
||||
def test_init_creates_logs_directory(tmp_path):
|
||||
"""Test that init creates logs directory."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
logs_dir = tmp_path / "logs"
|
||||
assert logs_dir.exists()
|
||||
assert logs_dir.is_dir()
|
||||
|
||||
|
||||
def test_init_creates_config_file(tmp_path):
|
||||
"""Test that init creates ArchiveBox.conf config file."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
config_file = tmp_path / "ArchiveBox.conf"
|
||||
assert config_file.exists()
|
||||
assert config_file.is_file()
|
||||
|
||||
|
||||
def test_init_runs_migrations(tmp_path):
|
||||
"""Test that init runs Django migrations and creates core tables."""
|
||||
run_archivebox_cmd(["init"])
|
||||
for relative_path in ("archive", "archive/users", "sources", "logs"):
|
||||
assert (tmp_path / relative_path).is_dir(), relative_path
|
||||
|
||||
with use_archivebox_db(tmp_path):
|
||||
migration_count = MigrationRecorder.Migration.objects.count()
|
||||
|
||||
assert migration_count > 0
|
||||
|
||||
|
||||
def test_init_creates_core_snapshot_table(tmp_path):
|
||||
"""Test that init creates core_snapshot table."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
assert Snapshot._meta.db_table == "core_snapshot"
|
||||
with use_archivebox_db(tmp_path):
|
||||
assert MigrationRecorder.Migration.objects.count() > 0
|
||||
assert Snapshot._meta.db_table == "core_snapshot"
|
||||
assert Snapshot.objects.count() == 0
|
||||
|
||||
|
||||
def test_init_creates_crawls_crawl_table(tmp_path):
|
||||
"""Test that init creates crawls_crawl table."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
assert Crawl._meta.db_table == "crawls_crawl"
|
||||
with use_archivebox_db(tmp_path):
|
||||
assert Crawl._meta.db_table == "crawls_crawl"
|
||||
assert Crawl.objects.count() == 0
|
||||
|
||||
|
||||
def test_init_creates_core_archiveresult_table(tmp_path):
|
||||
"""Test that init creates core_archiveresult table."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
assert ArchiveResult._meta.db_table == "core_archiveresult"
|
||||
with use_archivebox_db(tmp_path):
|
||||
assert ArchiveResult._meta.db_table == "core_archiveresult"
|
||||
assert ArchiveResult.objects.count() == 0
|
||||
assert Machine._meta.db_table == "machine_machine"
|
||||
Machine.objects.count()
|
||||
|
||||
|
||||
def test_init_sets_correct_file_permissions(tmp_path):
|
||||
"""Test that init sets correct permissions on created files."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
# Check database permissions
|
||||
db_path = tmp_path / "index.sqlite3"
|
||||
assert oct(db_path.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
|
||||
|
||||
# Check directory permissions
|
||||
archive_dir = tmp_path / "archive"
|
||||
assert oct(archive_dir.stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
|
||||
assert oct((tmp_path / "archive").stat().st_mode)[-3:] in (get_config().OUTPUT_PERMISSIONS, DIR_PERMISSIONS)
|
||||
output = result.stdout
|
||||
assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output
|
||||
|
||||
|
||||
def test_init_creates_database_with_archivebox_permissions_despite_permissive_umask(tmp_path):
|
||||
@@ -325,24 +248,6 @@ def test_init_quick_flag_skips_checks(tmp_path):
|
||||
assert db_path.exists()
|
||||
|
||||
|
||||
def test_init_creates_machine_table(tmp_path):
|
||||
"""Test that init creates the machine_machine table."""
|
||||
run_archivebox_cmd(["init"])
|
||||
|
||||
assert Machine._meta.db_table == "machine_machine"
|
||||
with use_archivebox_db(tmp_path):
|
||||
Machine.objects.count()
|
||||
|
||||
|
||||
def test_init_output_shows_collection_info(tmp_path):
|
||||
"""Test that init output shows helpful collection information."""
|
||||
result = run_archivebox_cmd(["init"])
|
||||
|
||||
output = result.stdout
|
||||
# Should show some helpful info about the collection
|
||||
assert "ArchiveBox" in output or "collection" in output.lower() or "Initializing" in output
|
||||
|
||||
|
||||
def test_init_ignores_unrecognized_archive_directories(initialized_archive):
|
||||
"""Test that init upgrades existing dirs without choking on extra folders."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
@@ -22,52 +22,14 @@ from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def test_install_runs_successfully(initialized_archive):
|
||||
"""Test that install command runs without error."""
|
||||
result = run_archivebox_cmd(
|
||||
["install", "--dry-run"],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "Dry run - would detect ArchiveBox dependencies" in result.stdout
|
||||
|
||||
|
||||
def test_install_creates_binary_records_in_db(initialized_archive):
|
||||
"""Test that install --dry-run does not create Binary records in database."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["install", "--dry-run"],
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Binary.objects.count() == 0
|
||||
|
||||
|
||||
def test_install_dry_run_does_not_install(initialized_archive):
|
||||
"""Test that --dry-run doesn't actually install anything."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["install", "--dry-run"],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
def test_install_dry_run_reports_plan_without_creating_binary_records(initialized_archive):
|
||||
"""Inspect one dry run; real installs and empty-dir bootstrap stay separate."""
|
||||
result = run_archivebox_cmd(["install", "--dry-run"], timeout=60)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert result.stdout.strip() == "Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow"
|
||||
|
||||
|
||||
def test_install_detects_system_binaries(initialized_archive):
|
||||
"""Test that install detects existing system binaries."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["install", "--dry-run"],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "ArchiveBox dependencies" in result.stdout
|
||||
assert "dry run" in result.stdout.lower()
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Binary.objects.count() == 0
|
||||
|
||||
|
||||
def test_install_target_resolver_uses_declared_binary_aliases():
|
||||
@@ -95,29 +57,6 @@ def test_default_dependency_install_memory_preflight_fails_before_installers(cap
|
||||
assert "No plugin dependency installers were started" in output
|
||||
|
||||
|
||||
def test_install_shows_binary_status(initialized_archive):
|
||||
"""Test that install shows status of binaries."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["install", "--dry-run"],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert result.stdout.strip() == "Dry run - would detect ArchiveBox dependencies and run the abx-dl install flow"
|
||||
|
||||
|
||||
def test_install_dry_run_prints_dry_run_message(initialized_archive):
|
||||
"""Test that install --dry-run clearly reports that no changes will be made."""
|
||||
result = run_archivebox_cmd(
|
||||
["install", "--dry-run"],
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "dry run" in result.stdout.lower()
|
||||
|
||||
|
||||
def test_install_help_lists_dry_run_flag(tmp_path):
|
||||
"""Test that install --help documents the dry-run option."""
|
||||
result = run_archivebox_cmd(
|
||||
|
||||
@@ -52,6 +52,7 @@ def test_persona_import_real_browser_session(initialized_archive, tmp_path, http
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from werkzeug.wrappers import Response
|
||||
from archivebox.tests.conftest import resolve_abxpkg_chrome_env
|
||||
@@ -167,7 +168,9 @@ const puppeteer = require(process.argv[1]);
|
||||
assert archived.returncode == 0, archived.stderr
|
||||
launch_commands = list((initialized_archive / "archive" / "users").glob("*/crawls/**/chrome/cmd.sh"))
|
||||
assert len(launch_commands) == 1
|
||||
assert shlex.split(launch_commands[0].read_text().splitlines()[1])[0] == browser_env["CHROME_BINARY"]
|
||||
launched_browser = Path(shlex.split(launch_commands[0].read_text().splitlines()[1])[0])
|
||||
# abxpkg's environment and provider projections can name the same binary.
|
||||
assert launched_browser.samefile(browser_env["CHROME_BINARY"])
|
||||
outputs = list((initialized_archive / "archive" / "users").glob("*/snapshots/**/dom/output.html"))
|
||||
assert len(outputs) == 1
|
||||
html = outputs[0].read_text()
|
||||
|
||||
@@ -29,8 +29,8 @@ print(json.dumps([
|
||||
return json.loads(result.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def test_remove_deletes_snapshot_from_db(initialized_archive):
|
||||
"""Test that remove command deletes snapshot from database."""
|
||||
def test_remove_yes_deletes_database_row_and_directory_without_confirmation(initialized_archive):
|
||||
"""Verify the DB, filesystem, exit code and summary of one confirmed removal."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
# Add a snapshot
|
||||
@@ -46,62 +46,17 @@ def test_remove_deletes_snapshot_from_db(initialized_archive):
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
|
||||
assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
|
||||
|
||||
# Remove it
|
||||
run_archivebox_cmd(
|
||||
["remove", "https://example.com", "--yes"],
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert len(_snapshot_rows(initialized_archive, env)) == 0
|
||||
assert not snapshot_dir.exists()
|
||||
|
||||
|
||||
def test_remove_deletes_archive_directory(initialized_archive):
|
||||
"""Test that remove --yes removes the current snapshot output directory."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
# Add a snapshot
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
env=env,
|
||||
)
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
|
||||
rows = _snapshot_rows(initialized_archive, env)
|
||||
assert len(rows) == 1
|
||||
snapshot_id = rows[0]["id"]
|
||||
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
|
||||
assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
|
||||
|
||||
run_archivebox_cmd(
|
||||
["remove", "https://example.com", "--yes"],
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert not snapshot_dir.exists()
|
||||
|
||||
|
||||
def test_remove_yes_flag_skips_confirmation(initialized_archive):
|
||||
"""Test that --yes flag skips confirmation prompt."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
env=env,
|
||||
)
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
|
||||
# Remove with --yes should complete without interaction
|
||||
# Remove with --yes must finish without interaction.
|
||||
result = run_archivebox_cmd(
|
||||
["remove", "https://example.com", "--yes"],
|
||||
env=env,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
output = result.stdout + result.stderr
|
||||
assert "Index now contains 0 links." in output
|
||||
assert "Index now contains 0 links." in result.stdout + result.stderr
|
||||
|
||||
assert len(_snapshot_rows(initialized_archive, env)) == 0
|
||||
assert not snapshot_dir.exists()
|
||||
|
||||
|
||||
def test_remove_without_yes_prompts_and_keeps_snapshot(initialized_archive):
|
||||
|
||||
@@ -177,70 +177,41 @@ def test_sqlite_connections_use_explicit_busy_timeout():
|
||||
assert "PRAGMA journal_mode = WAL;" in SQLITE_CONNECTION_OPTIONS["OPTIONS"]["init_command"]
|
||||
|
||||
|
||||
def test_docker_sqlite_never_uses_wal_on_host_shared_collection(tmp_path):
|
||||
"""Docker collections must not expose a WAL database through /data.
|
||||
def test_sqlite_add_commits_while_read_cursor_is_open(tmp_path):
|
||||
"""A real read snapshot must not hold up a new public CLI submission."""
|
||||
import sqlite3
|
||||
|
||||
Docker Desktop/OrbStack bind mounts cross the Linux VM/host locking
|
||||
boundary. A container WAL writer plus a host-side sqlite reader can make
|
||||
the writer SIGBUS and leave index.sqlite3 malformed; the reader does not
|
||||
need to write. Run a fresh real settings process with Docker identity so a
|
||||
host test cannot accidentally pass by reusing this process's non-Docker
|
||||
config imports.
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
env["IN_DOCKER"] = "True"
|
||||
env["DJANGO_SETTINGS_MODULE"] = "archivebox.core.settings"
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
(
|
||||
"import django;"
|
||||
"django.setup();"
|
||||
"from django.db import connection;"
|
||||
"cursor=connection.cursor();"
|
||||
"cursor.execute('CREATE TABLE journal_probe (value INTEGER)');"
|
||||
"cursor.execute('INSERT INTO journal_probe VALUES (1)');"
|
||||
"cursor.execute('PRAGMA journal_mode');"
|
||||
"print(cursor.fetchone()[0]);"
|
||||
"cursor.close();"
|
||||
"connection.close()"
|
||||
),
|
||||
],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
env = {"SQLITE_JOURNAL_MODE": "WAL"}
|
||||
initialized = run_archivebox_cmd(["init", "--quick"], cwd=tmp_path, env=env, timeout=90)
|
||||
assert initialized.returncode == 0, initialized.stderr or initialized.stdout
|
||||
for url in ("https://example.com/read-one", "https://example.com/read-two"):
|
||||
initial = run_archivebox_cmd(["add", "--index-only", url], cwd=tmp_path, env=env, timeout=90)
|
||||
assert initial.returncode == 0, initial.stderr or initial.stdout
|
||||
|
||||
assert result.stdout.strip().lower() == "delete"
|
||||
assert not tmp_path.joinpath("index.sqlite3-wal").exists()
|
||||
assert not tmp_path.joinpath("index.sqlite3-shm").exists()
|
||||
|
||||
|
||||
def test_docker_rejects_explicit_wal_override(tmp_path):
|
||||
"""An old config or environment override must not bypass the invariant."""
|
||||
env = os.environ.copy()
|
||||
env["IN_DOCKER"] = "True"
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-c",
|
||||
("from archivebox.config.common import DatabaseConfig;DatabaseConfig(SQLITE_JOURNAL_MODE='WAL')"),
|
||||
],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "WAL is unsafe for Docker collections" in result.stderr
|
||||
reader = sqlite3.connect(f"file:{tmp_path / 'index.sqlite3'}?mode=ro", uri=True)
|
||||
try:
|
||||
assert reader.execute("PRAGMA journal_mode").fetchone()[0] == "wal"
|
||||
cursor = reader.execute("SELECT urls FROM crawls_crawl ORDER BY urls")
|
||||
first = cursor.fetchone()
|
||||
assert first == ("https://example.com/read-one",)
|
||||
# Do not exhaust/close the actual cursor until add commits. This is the
|
||||
# contention that rollback journaling turns into seconds of waiting.
|
||||
added = run_archivebox_cmd(
|
||||
["add", "--index-only", "https://example.com/write-during-read"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=30,
|
||||
)
|
||||
assert added.returncode == 0, added.stderr or added.stdout
|
||||
assert cursor.fetchall() == [("https://example.com/read-two",)]
|
||||
cursor.close()
|
||||
assert reader.execute("SELECT urls FROM crawls_crawl WHERE urls = ?", ("https://example.com/write-during-read",)).fetchone()
|
||||
finally:
|
||||
reader.close()
|
||||
|
||||
|
||||
def test_docker_postgres_ignores_irrelevant_sqlite_wal_override(tmp_path):
|
||||
"""The Docker SQLite safety invariant must not reject PostgreSQL.
|
||||
"""SQLite filesystem constraints must not reject PostgreSQL.
|
||||
|
||||
Operators can switch an existing deployment to PostgreSQL while an old
|
||||
SQLITE_JOURNAL_MODE setting remains in ArchiveBox.conf or the environment.
|
||||
|
||||
@@ -24,21 +24,17 @@ def _create_snapshot_rows(initialized_archive, env, *urls):
|
||||
return result
|
||||
|
||||
|
||||
def test_status_runs_successfully(initialized_archive):
|
||||
"""Test that status command runs without error."""
|
||||
def test_status_reports_empty_collection_size_user_index_and_path(initialized_archive):
|
||||
"""All empty-collection fields come from the same real status response."""
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert len(result.stdout) > 100
|
||||
|
||||
|
||||
def test_status_shows_zero_snapshots_in_empty_archive(initialized_archive):
|
||||
"""Test status shows 0 snapshots in empty archive."""
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
output = result.stdout
|
||||
# Should indicate empty/zero state
|
||||
assert len(output) > 100
|
||||
assert "0" in output
|
||||
assert "Size" in output or "size" in output
|
||||
assert "user" in output.lower() or "login" in output.lower()
|
||||
assert "index" in output.lower() or "Index" in output
|
||||
assert "archive" in output.lower() or str(initialized_archive) in output
|
||||
|
||||
|
||||
def test_status_shows_correct_snapshot_count(initialized_archive):
|
||||
@@ -58,36 +54,16 @@ def test_status_shows_correct_snapshot_count(initialized_archive):
|
||||
assert "3" in result.stdout
|
||||
|
||||
|
||||
def test_status_shows_archived_count(initialized_archive):
|
||||
"""Test status distinguishes archived vs unarchived snapshots."""
|
||||
def test_status_reports_database_snapshot_count_and_archive_categories(initialized_archive):
|
||||
"""One uncaptured snapshot exercises the DB count and filesystem categories."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
_create_snapshot_rows(initialized_archive, env, "https://example.com")
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Snapshot.objects.count() == 1
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
# Should show archived/unarchived categories
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "1" in result.stdout
|
||||
assert "archived" in result.stdout.lower() or "queued" in result.stdout.lower()
|
||||
|
||||
|
||||
def test_status_shows_archive_directory_size(initialized_archive):
|
||||
"""Test status reports archive directory size."""
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
output = result.stdout
|
||||
# Should show size info
|
||||
assert "Size" in output or "size" in output
|
||||
|
||||
|
||||
def test_status_counts_archive_directories(initialized_archive):
|
||||
"""Test status counts directories in archive/ folder."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
_create_snapshot_rows(initialized_archive, env, "https://example.com")
|
||||
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
# Should show directory count
|
||||
assert "present" in result.stdout.lower() or "directories" in result.stdout
|
||||
|
||||
|
||||
@@ -131,40 +107,6 @@ def test_status_counts_new_snapshot_output_dirs_as_archived(initialized_archive,
|
||||
assert "present: 1" in result.stdout
|
||||
|
||||
|
||||
def test_status_shows_user_info(initialized_archive):
|
||||
"""Test status shows user/login information."""
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
output = result.stdout
|
||||
# Should show user section
|
||||
assert "user" in output.lower() or "login" in output.lower()
|
||||
|
||||
|
||||
def test_status_reads_from_db_not_filesystem(initialized_archive):
|
||||
"""Test that status uses DB as source of truth, not filesystem."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
_create_snapshot_rows(initialized_archive, env, "https://example.com")
|
||||
|
||||
# Verify DB has snapshot
|
||||
with use_archivebox_db(initialized_archive):
|
||||
db_count = Snapshot.objects.count()
|
||||
|
||||
assert db_count == 1
|
||||
|
||||
# Status should reflect DB count
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
assert "1" in result.stdout
|
||||
|
||||
|
||||
def test_status_shows_index_file_info(initialized_archive):
|
||||
"""Test status shows index file information."""
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
# Should mention index
|
||||
assert "index" in result.stdout.lower() or "Index" in result.stdout
|
||||
|
||||
|
||||
def test_status_help_lists_available_options(initialized_archive):
|
||||
"""Test that status --help works and documents the command."""
|
||||
result = run_archivebox_cmd(
|
||||
@@ -174,10 +116,3 @@ def test_status_help_lists_available_options(initialized_archive):
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "status" in result.stdout.lower() or "statistic" in result.stdout.lower()
|
||||
|
||||
|
||||
def test_status_shows_data_directory_path(initialized_archive):
|
||||
"""Test that status reports which collection directory it is inspecting."""
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive)
|
||||
|
||||
assert "archive" in result.stdout.lower() or str(initialized_archive) in result.stdout
|
||||
@@ -18,6 +18,50 @@ from pathlib import Path
|
||||
from archivebox.config.paths import tmp_dir_socket_path_is_short_enough
|
||||
from archivebox.cli.archivebox_version import _binary_row_dedupe_key
|
||||
from archivebox.tests.conftest import cli_env, run_archivebox_cmd
|
||||
from abx_plugins.plugins.opencode.archivebox.host_cases import (
|
||||
test_version_names_disabling_config_in_path_column as test_version_names_disabling_config_in_path_column,
|
||||
test_version_shared_git_stays_installed_when_opencode_disabled as test_version_shared_git_stays_installed_when_opencode_disabled,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_cache", [False, True])
|
||||
def test_version_reuses_validated_provider_cache(tmp_path, no_cache):
|
||||
"""A warm diagnostic must validate the install without hashing it again."""
|
||||
env = cli_env(ABXPKG_LIB_DIR=str(tmp_path / "lib"), COLUMNS="500")
|
||||
first = run_archivebox_cmd(["version", "--binaries=git"], cwd=tmp_path, env=env)
|
||||
assert first.returncode == 0, first.stdout + first.stderr
|
||||
assert "✅" in first.stdout
|
||||
|
||||
# Observe the real CLI and its resolver threads; no replacement providers or
|
||||
# timing thresholds. This distinguishes validated cache reads from a full
|
||||
# rehash even on a fast machine with a small git executable.
|
||||
script = """
|
||||
import runpy, sys, threading
|
||||
calls = []
|
||||
def observe(frame, event, arg):
|
||||
if event == 'call' and frame.f_code.co_name == 'get_sha256':
|
||||
calls.append(frame.f_code.co_filename)
|
||||
sys.setprofile(observe)
|
||||
threading.setprofile(observe)
|
||||
sys.argv = ['archivebox', 'version', '--binaries=git', *sys.argv[1:]]
|
||||
try:
|
||||
runpy.run_module('archivebox', run_name='__main__')
|
||||
finally:
|
||||
sys.setprofile(None)
|
||||
threading.setprofile(None)
|
||||
print('BINARY_HASH_CALLS=' + str(len(calls)))
|
||||
"""
|
||||
second = subprocess.run(
|
||||
["uv", "run", "--active", "python", "-c", script, *(["--no-cache"] if no_cache else [])],
|
||||
cwd=tmp_path,
|
||||
env={**os.environ, **env},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
)
|
||||
assert second.returncode == 0, second.stdout + second.stderr
|
||||
assert "✅" in second.stdout
|
||||
assert f"BINARY_HASH_CALLS={int(no_cache)}" in second.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [80, 160])
|
||||
@@ -243,30 +287,6 @@ def test_version_includes_disabled_plugin_dependencies(tmp_path):
|
||||
assert "Failed to detect the following binaries" not in output
|
||||
|
||||
|
||||
def test_version_names_disabling_config_in_path_column(tmp_path):
|
||||
env = cli_env(OPENCODE_ENABLED="False", COLUMNS="200")
|
||||
result = run_archivebox_cmd(["version", "--binaries=opencode"], cwd=tmp_path, env=env)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
rows = [re.split(r"\s{2,}", line.strip()) for line in result.stdout.splitlines()]
|
||||
opencode_rows = [row for row in rows if len(row) == 6 and row[1] == "opencode"]
|
||||
assert len(opencode_rows) == 4, result.stdout
|
||||
assert all(row[2] == "disabled" and row[5] == "disabled by OPENCODE_ENABLED=False" for row in opencode_rows)
|
||||
assert "Disabled plugins are dimmed" not in result.stdout
|
||||
|
||||
|
||||
def test_version_shared_git_stays_installed_when_opencode_disabled(tmp_path):
|
||||
env = cli_env(PLUGINS="git", GIT_ENABLED="True", OPENCODE_ENABLED="False", COLUMNS="300")
|
||||
result = run_archivebox_cmd(["version", "--binaries=git"], cwd=tmp_path, env=env)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
rows = [re.split(r"\s{2,}", line.strip()) for line in result.stdout.splitlines()]
|
||||
git_rows = [row for row in rows if len(row) == 6 and row[0] == "git"]
|
||||
assert len(git_rows) == 1, result.stdout
|
||||
assert set(git_rows[0][1].split(", ")) == {"git", "opencode"}
|
||||
assert git_rows[0][2] == "✅"
|
||||
assert Path(git_rows[0][5]).is_file()
|
||||
assert "disabled by OPENCODE_ENABLED=False" not in result.stdout
|
||||
|
||||
|
||||
def test_version_honors_legacy_save_aliases_when_disabling_extractors(tmp_path):
|
||||
"""Legacy SAVE_* aliases should disable their canonical plugin flags."""
|
||||
data_dir = tmp_path / "legacy-save-disabled"
|
||||
|
||||
@@ -159,7 +159,7 @@ print(json.dumps({{
|
||||
assert not Path(retained["archiveresult_dir"]).exists()
|
||||
|
||||
|
||||
def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
def test_delete_after_real_add_page_and_rest_create_paths(client, recursive_test_site):
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
# Match init/server startup; visiting /add/ must not create persona records.
|
||||
@@ -172,6 +172,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
)
|
||||
|
||||
client.force_login(admin_user)
|
||||
ui_url, rest_url = recursive_test_site["child_urls"][:2]
|
||||
response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST)
|
||||
assert response.status_code == 200
|
||||
assert 'name="delete_after"' in response.content.decode()
|
||||
@@ -179,7 +180,7 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
response = client.post(
|
||||
reverse("add"),
|
||||
data={
|
||||
"url": "https://example.com/delete-after-ui",
|
||||
"url": ui_url,
|
||||
"tag": "retention-ui",
|
||||
"depth": "0",
|
||||
"max_urls": "1",
|
||||
@@ -203,13 +204,13 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
ui_crawl = Crawl.objects.get(urls__contains="https://example.com/delete-after-ui")
|
||||
ui_crawl = Crawl.objects.get(urls=ui_url)
|
||||
assert ui_crawl.config["DELETE_AFTER"] == "2h"
|
||||
assert ui_crawl.delete_at is not None
|
||||
from archivebox.services.runner import run_due_crawl
|
||||
|
||||
assert run_due_crawl(ui_crawl, lock_seconds=10)
|
||||
ui_snapshot = ui_crawl.snapshot_set.get(url="https://example.com/delete-after-ui")
|
||||
ui_snapshot = ui_crawl.snapshot_set.get(url=ui_url)
|
||||
assert ui_snapshot.delete_at is not None
|
||||
assert not ui_snapshot.output_dir.joinpath("staticfile", "stdin.txt").exists()
|
||||
|
||||
@@ -221,10 +222,12 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
"/api/v1/crawls/crawls",
|
||||
data=json.dumps(
|
||||
{
|
||||
"urls": ["https://example.com/delete-after-rest"],
|
||||
"urls": [rest_url],
|
||||
"max_depth": 0,
|
||||
"max_urls": 1,
|
||||
"config": {"DELETE_AFTER": "3h"},
|
||||
# Exercise retention through a real capture. The full plugin
|
||||
# pipeline has its own recursive-crawl integration coverage.
|
||||
"config": {"DELETE_AFTER": "3h", "PLUGINS": "wget,hashes"},
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
@@ -232,9 +235,16 @@ def test_delete_after_real_add_page_and_rest_create_paths(client):
|
||||
HTTP_X_ARCHIVEBOX_API_KEY=api_token.token,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
rest_crawl = Crawl.objects.get(urls__contains="https://example.com/delete-after-rest")
|
||||
rest_crawl = Crawl.objects.get(urls=rest_url)
|
||||
assert rest_crawl.config["DELETE_AFTER"] == "3h"
|
||||
assert rest_crawl.delete_at is not None
|
||||
assert run_due_crawl(rest_crawl, lock_seconds=10)
|
||||
rest_snapshot = rest_crawl.snapshot_set.get(url="https://example.com/delete-after-rest")
|
||||
rest_snapshot = rest_crawl.snapshot_set.get(url=rest_url)
|
||||
assert rest_snapshot.delete_at is not None
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
for plugin in ("wget", "hashes"):
|
||||
result = rest_snapshot.archiveresult_set.get(plugin=plugin)
|
||||
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result.delete_at is not None
|
||||
assert result.output_size > 0
|
||||
@@ -218,13 +218,6 @@ class TestRequiredBinaryConfigHandling:
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "Wget" in result.stdout
|
||||
|
||||
def test_binary_env_var_empty_default(self):
|
||||
"""The shipped wget schema should retain wget as its required binary."""
|
||||
config = json.loads(files("abx_plugins.plugins.wget").joinpath("config.json").read_text())
|
||||
|
||||
assert config["required_binaries"][0]["name"] == "{WGET_BINARY}"
|
||||
assert config["properties"]["WGET_BINARY"]["default"] == "wget"
|
||||
|
||||
|
||||
class TestHookDiscovery:
|
||||
"""Test hook discovery functions."""
|
||||
@@ -455,38 +448,6 @@ class TestHookExecution:
|
||||
assert process.exit_code == 0, process.stderr
|
||||
assert "chrome zombies" in process.stdout
|
||||
|
||||
def test_hook_receives_cli_args(self, tmp_path):
|
||||
"""Hook should receive CLI arguments."""
|
||||
snap_dir = tmp_path / "snapshot"
|
||||
output_dir = snap_dir / "hashes"
|
||||
output_dir.mkdir(parents=True)
|
||||
(snap_dir / "source.txt").write_text("real CLI argument input", encoding="utf-8")
|
||||
hook_path = Path(
|
||||
str(
|
||||
files("abx_plugins.plugins.hashes").joinpath(
|
||||
"on_Snapshot__93_hashes.py",
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[str(hook_path), "--url=https://example.com/real-hook-argument"],
|
||||
cwd=output_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={**os.environ, "SNAP_DIR": str(snap_dir)},
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
records = Process.parse_records_from_text(result.stdout)
|
||||
source_hash = hashlib.sha256(b"real CLI argument input").hexdigest()
|
||||
assert records == [{"type": "ArchiveResult", "status": "succeeded", "output_str": f"0.0MB {source_hash[:12]}"}]
|
||||
hashes = json.loads((output_dir / "hashes.json").read_text())
|
||||
assert hashes["files"][0]["hash"] == source_hash
|
||||
|
||||
|
||||
class TestDependencyRecordOutput:
|
||||
"""Test Binary JSONL emitted by the real CLI and persisted model."""
|
||||
|
||||
@@ -312,6 +312,15 @@ def test_migration_preserves_08_collection(migration_08_data):
|
||||
extractors = {ar["extractor"] for ar in original_data["archiveresults"]}
|
||||
assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}"
|
||||
|
||||
# Legacy convergence created Binary with raw SQL and omitted the declared
|
||||
# queue index. A migrated archive must not scan all installed binaries on
|
||||
# every scheduler pass, even though its migration state says db_index=True.
|
||||
binary_indexes = cursor.execute("PRAGMA index_list(machine_binary)").fetchall()
|
||||
indexed_columns = {
|
||||
tuple(row[2] for row in cursor.execute("SELECT * FROM pragma_index_info(?)", (index[1],)).fetchall()) for index in binary_indexes
|
||||
}
|
||||
assert ("retry_at",) in indexed_columns
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*)
|
||||
FROM machine_process
|
||||
|
||||
@@ -1,291 +1,11 @@
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
"""Run plugin-owned host integration cases in the ArchiveBox CI matrix."""
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
def _reset_runtime_config() -> None:
|
||||
from archivebox.config import common
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
for value in vars(common).values():
|
||||
cache_clear = getattr(value, "cache_clear", None)
|
||||
if cache_clear is not None:
|
||||
cache_clear()
|
||||
Machine.current()
|
||||
|
||||
|
||||
def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
|
||||
os.chdir(data_dir)
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", *values],
|
||||
cwd=data_dir,
|
||||
env=env,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
_reset_runtime_config()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opencode_archive_config(initialized_archive):
|
||||
port = _free_port()
|
||||
state_dir = initialized_archive / "opencode"
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
|
||||
"OPENCODE_ENABLED": "True",
|
||||
"OPENCODE_HOST": "127.0.0.1",
|
||||
"OPENCODE_PORT": str(port),
|
||||
"OPENCODE_WORKDIR": str(initialized_archive),
|
||||
"OPENCODE_STATE_DIR": str(state_dir),
|
||||
"OPENCODE_TIMEOUT": "60",
|
||||
},
|
||||
)
|
||||
_set_archivebox_config(
|
||||
initialized_archive,
|
||||
"OPENCODE_ENABLED=True",
|
||||
"OPENCODE_HOST=127.0.0.1",
|
||||
f"OPENCODE_PORT={port}",
|
||||
f"OPENCODE_WORKDIR={initialized_archive}",
|
||||
f"OPENCODE_STATE_DIR={state_dir}",
|
||||
"OPENCODE_TIMEOUT=60",
|
||||
env=env,
|
||||
)
|
||||
return SimpleNamespace(data_dir=initialized_archive, port=port, state_dir=state_dir, env=env)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def installed_opencode(opencode_archive_config):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
install = run_archivebox_cmd(
|
||||
["install", "opencode", "--binproviders=env,pnpm"],
|
||||
cwd=opencode_archive_config.data_dir,
|
||||
env=opencode_archive_config.env,
|
||||
timeout=1200,
|
||||
)
|
||||
assert install.returncode == 0, install.stderr or install.stdout
|
||||
_reset_runtime_config()
|
||||
|
||||
config = get_config().model_dump(mode="json")
|
||||
settings = runtime._settings(config, opencode_archive_config.data_dir)
|
||||
settings["archivebox_base_url"] = "http://admin.archivebox.localhost:5797"
|
||||
settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:5797/admin"
|
||||
settings["archivebox_api_url"] = "http://admin.archivebox.localhost:5797/api/"
|
||||
binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"])
|
||||
version = binary.exec(
|
||||
cmd=("--version",),
|
||||
env={**os.environ, **binary_env},
|
||||
timeout=120,
|
||||
)
|
||||
assert version.returncode == 0, version.stderr or version.stdout
|
||||
return SimpleNamespace(config=opencode_archive_config, settings=settings)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_opencode(installed_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
settings = installed_opencode.settings
|
||||
ok, error = runtime._ensure_opencode(settings)
|
||||
assert ok, error
|
||||
|
||||
process = runtime._PROCESS
|
||||
assert process is not None
|
||||
try:
|
||||
yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process)
|
||||
finally:
|
||||
runtime._stop_owned_process()
|
||||
|
||||
|
||||
def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive):
|
||||
_set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False")
|
||||
|
||||
assert get_config().OPENCODE_ENABLED is False
|
||||
assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404
|
||||
for path in ("/add/", "/admin/core/snapshot/"):
|
||||
response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200
|
||||
assert b'href="/admin/agent"' not in response.content
|
||||
|
||||
|
||||
def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_opencode):
|
||||
response = admin_client.post(
|
||||
"/admin/agent/opencode/session",
|
||||
data=b"{}",
|
||||
content_type="application/json",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="cross-site",
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_proxy_waits_for_owned_process_readiness(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
process = runtime._PROCESS
|
||||
assert process is not None
|
||||
runtime._PROCESS_READY = None
|
||||
workdir = quote(str(live_opencode.config.data_dir.resolve()))
|
||||
|
||||
response = admin_client.get(
|
||||
f"/admin/agent/opencode/path?directory={workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert runtime._PROCESS is process
|
||||
assert runtime._PROCESS_READY is process
|
||||
|
||||
|
||||
def test_opencode_proxy_preserves_protocol_headers(admin_client, live_opencode):
|
||||
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
|
||||
created = admin_client.post(
|
||||
"/admin/agent/opencode/pty",
|
||||
data={"command": "/bin/sh", "args": []},
|
||||
content_type="application/json",
|
||||
**headers,
|
||||
)
|
||||
assert created.status_code == 200
|
||||
path = f"/admin/agent/opencode/pty/{created.json()['id']}"
|
||||
try:
|
||||
response = admin_client.post(path + "/connect-token", HTTP_X_OPENCODE_TICKET="1", **headers)
|
||||
assert response.status_code == 200, response.content
|
||||
assert response.json()["ticket"]
|
||||
finally:
|
||||
deleted = admin_client.delete(path, **headers)
|
||||
assert deleted.status_code == 200, deleted.content
|
||||
|
||||
|
||||
def test_opencode_starts_with_isolated_state(admin_client, live_opencode):
|
||||
workdir = str(live_opencode.config.data_dir.resolve())
|
||||
state_dir = live_opencode.config.state_dir
|
||||
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert agent.status_code == 200
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
frame = admin_client.get(
|
||||
runtime._project_route(live_opencode.config.data_dir),
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
)
|
||||
assert frame.status_code == 302
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
|
||||
project = requests.get(
|
||||
f"{live_opencode.settings['origin']}/project/current",
|
||||
params={"directory": workdir},
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
project.raise_for_status()
|
||||
|
||||
config = requests.get(
|
||||
f"{live_opencode.settings['origin']}/global/config",
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
config.raise_for_status()
|
||||
|
||||
assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
|
||||
assert project.json()["id"] == "global"
|
||||
assert not project.json().get("vcs")
|
||||
assert not config.json().get("model")
|
||||
assert config.json()["snapshot"] is False
|
||||
assert live_opencode.process.poll() is None
|
||||
path = requests.get(
|
||||
f"{live_opencode.settings['origin']}/path",
|
||||
params={"directory": workdir},
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
path.raise_for_status()
|
||||
assert Path(path.json()["directory"]).resolve() == Path(workdir)
|
||||
|
||||
diff = requests.get(
|
||||
f"{live_opencode.settings['origin']}/vcs/diff",
|
||||
params={"directory": workdir, "mode": "git"},
|
||||
timeout=5,
|
||||
)
|
||||
diff.raise_for_status()
|
||||
assert diff.json() == []
|
||||
assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
|
||||
assert (state_dir / "SKILL.md").is_file()
|
||||
assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("damaged_file", "missing"),
|
||||
[
|
||||
("runtime.py", True),
|
||||
("templates/agent.html", True),
|
||||
("templates/agent.html", False),
|
||||
("templates/navigation.html", False),
|
||||
("templates/add.html", False),
|
||||
],
|
||||
from abx_plugins.plugins.opencode.archivebox.agent_1_cases import (
|
||||
pytestmark as pytestmark,
|
||||
test_opencode_disabled_via_cli_stays_disabled as test_opencode_disabled_via_cli_stays_disabled,
|
||||
test_opencode_proxy_blocks_cross_site_fetch_metadata as test_opencode_proxy_blocks_cross_site_fetch_metadata,
|
||||
test_opencode_proxy_waits_for_owned_process_readiness as test_opencode_proxy_waits_for_owned_process_readiness,
|
||||
test_opencode_proxy_preserves_protocol_headers as test_opencode_proxy_preserves_protocol_headers,
|
||||
test_opencode_starts_with_isolated_state as test_opencode_starts_with_isolated_state,
|
||||
test_opencode_incomplete_install_does_not_break_archivebox as test_opencode_incomplete_install_does_not_break_archivebox,
|
||||
)
|
||||
def test_opencode_incomplete_install_does_not_break_archivebox(installed_opencode, tmp_path, damaged_file, missing):
|
||||
import abx_plugins
|
||||
|
||||
# Exercise a genuinely incomplete installation in a separate process;
|
||||
# never alter the shared package or intercept Python imports.
|
||||
site = tmp_path / "site"
|
||||
installed = site / "abx_plugins"
|
||||
shutil.copytree(Path(abx_plugins.__file__).parent, installed, ignore=shutil.ignore_patterns("__pycache__", "tests"))
|
||||
damaged_path = installed / "plugins" / "opencode" / damaged_file
|
||||
if missing:
|
||||
damaged_path.unlink()
|
||||
else:
|
||||
damaged_path.write_text("{% invalid_template_tag %}")
|
||||
expected_status = 200 if damaged_file in {"templates/navigation.html", "templates/add.html"} else 503
|
||||
script = f"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import abx_plugins
|
||||
from django.test import Client
|
||||
from django.contrib.auth import get_user_model
|
||||
assert Path(abx_plugins.__file__).is_relative_to({str(site)!r})
|
||||
user = get_user_model().objects.create_superuser(username='optional-service-test')
|
||||
client = Client(HTTP_HOST={ADMIN_TEST_HOST!r})
|
||||
client.force_login(user)
|
||||
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
|
||||
assert client.get(path).status_code == 200, path
|
||||
assert 'abx_plugins.plugins.opencode.runtime' not in sys.modules
|
||||
response = client.get('/admin/agent')
|
||||
assert response.status_code == {expected_status}, response.status_code
|
||||
if response.status_code == 503:
|
||||
assert response.content == b'AI service unavailable. See server logs.'
|
||||
if {damaged_file != "runtime.py"!r}:
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
assert runtime._PROCESS is None
|
||||
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
|
||||
assert client.get(path).status_code == 200, path
|
||||
print('OPTIONAL_SERVICE_FAILURE_ISOLATED')
|
||||
"""
|
||||
result = run_archivebox_cmd(
|
||||
["shell", "-c", script],
|
||||
cwd=installed_opencode.config.data_dir,
|
||||
env={**installed_opencode.config.env, "PYTHONPATH": str(site)},
|
||||
timeout=90,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "OPTIONAL_SERVICE_FAILURE_ISOLATED" in result.stdout
|
||||
@@ -1,251 +1,12 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FutureTimeoutError
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
"""Run plugin-owned host integration cases in the ArchiveBox CI matrix."""
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from asgiref.testing import ApplicationCommunicator
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
|
||||
from .test_opencode_agent_1 import (
|
||||
from abx_plugins.plugins.opencode.archivebox.agent_2_cases import (
|
||||
pytestmark as pytestmark,
|
||||
_free_port as _free_port,
|
||||
_reset_runtime_config as _reset_runtime_config,
|
||||
_set_archivebox_config as _set_archivebox_config,
|
||||
opencode_archive_config as opencode_archive_config,
|
||||
installed_opencode as installed_opencode,
|
||||
live_opencode as live_opencode,
|
||||
test_opencode_disabled_route_does_not_start_server as test_opencode_disabled_route_does_not_start_server,
|
||||
test_opencode_agent_requires_superuser_when_enabled as test_opencode_agent_requires_superuser_when_enabled,
|
||||
test_opencode_agent_superuser_gets_admin_wrapper as test_opencode_agent_superuser_gets_admin_wrapper,
|
||||
test_opencode_static_assets_cache_privately_and_revalidate as test_opencode_static_assets_cache_privately_and_revalidate,
|
||||
test_opencode_proxy_sse_response_is_unbuffered as test_opencode_proxy_sse_response_is_unbuffered,
|
||||
test_opencode_proxy_sse_returns_headers_before_restart_finishes as test_opencode_proxy_sse_returns_headers_before_restart_finishes,
|
||||
test_opencode_invalid_state_does_not_break_archivebox as test_opencode_invalid_state_does_not_break_archivebox,
|
||||
)
|
||||
|
||||
|
||||
def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
os.chdir(initialized_archive)
|
||||
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
|
||||
_reset_runtime_config()
|
||||
assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False
|
||||
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None
|
||||
|
||||
|
||||
def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode):
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 302
|
||||
assert "/admin/login/" in response.headers["Location"]
|
||||
|
||||
next_path = "/admin/agent?x=1&next=https://example.com"
|
||||
response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 302
|
||||
assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]}
|
||||
|
||||
user = django_user_model.objects.create_user(username="regular", password="testpassword")
|
||||
client.force_login(user)
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
frame_path = runtime._project_route(live_opencode.config.data_dir)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert f'<iframe data-src="{frame_path}"'.encode() in response.content
|
||||
assert b'id="opencode-agent-welcome"' in response.content
|
||||
assert b'id="header"' in response.content
|
||||
assert b'id="progress-monitor"' in response.content
|
||||
assert b'<a href="/admin/agent" class="navbar-item navbar-ai">' in response.content
|
||||
add_page = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert add_page.status_code == 200
|
||||
assert '<a href="/admin/agent">💬 Crawl with AI</a>'.encode() in add_page.content
|
||||
assert response.context["proxy_prefix"] == runtime._PROXY_PREFIX
|
||||
assert b"/_archivebox/health" not in response.content
|
||||
assert b"window.setInterval(check, 3000)" not in response.content
|
||||
assert response.headers["X-Frame-Options"] == "DENY"
|
||||
assert response.headers["Content-Security-Policy"] == "frame-ancestors 'none'"
|
||||
|
||||
frame = admin_client.get(
|
||||
frame_path,
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert frame.status_code == 302
|
||||
assert frame.headers["Location"].startswith(frame_path + "/")
|
||||
session = admin_client.get(
|
||||
frame.headers["Location"],
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert session.status_code == 200
|
||||
assert session.headers["X-Frame-Options"] == "SAMEORIGIN"
|
||||
assert session.headers["Content-Security-Policy"] == "frame-ancestors 'self'"
|
||||
|
||||
|
||||
def test_opencode_oauth_callback_waits_for_user_and_preserves_cancellation(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
settings = {**live_opencode.settings, "timeout": 1}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
status, _, body = runtime.proxy(settings, "POST", "provider/openai/oauth/authorize", (), headers, b'{"method":0}')
|
||||
assert status == 200
|
||||
authorization = json.loads(body)
|
||||
assert urlsplit(authorization["url"]).hostname == "auth.openai.com"
|
||||
redirect = parse_qs(urlsplit(authorization["url"]).query)["redirect_uri"][0]
|
||||
callback_origin = urlsplit(redirect)
|
||||
assert callback_origin.hostname == "localhost"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
callback = executor.submit(runtime.proxy, settings, "POST", "provider/openai/oauth/callback", (), headers, b'{"method":0}')
|
||||
try:
|
||||
# A real pending authorization must outlive the ordinary API read
|
||||
# timeout. No tokens or substituted provider responses are used.
|
||||
with pytest.raises(FutureTimeoutError):
|
||||
callback.result(timeout=2)
|
||||
finally:
|
||||
cancelled = requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=5)
|
||||
assert cancelled.status_code == 200
|
||||
assert cancelled.text == "Login cancelled"
|
||||
status, response_headers, body = callback.result(timeout=5)
|
||||
assert status == 500
|
||||
assert response_headers["Content-Type"].startswith("application/json")
|
||||
error = json.loads(body)
|
||||
assert error["name"] == "UnknownError"
|
||||
assert error["data"]["ref"].startswith("err_")
|
||||
runtime._stop_owned_process()
|
||||
# The pnpm launcher can exit before its server child. Stopping the owned
|
||||
# process must release both listeners so another collection can authorize.
|
||||
with pytest.raises(requests.ConnectionError):
|
||||
requests.get(settings["origin"] + "/global/health", timeout=2)
|
||||
with pytest.raises(requests.ConnectionError):
|
||||
requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=2)
|
||||
|
||||
|
||||
def test_opencode_static_assets_cache_privately_and_revalidate(admin_client, live_opencode):
|
||||
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
|
||||
page = admin_client.get("/admin/agent/opencode/", **headers)
|
||||
assert page.status_code == 200
|
||||
asset = re.search(rb'src="(/admin/agent/opencode/assets/[^\"]+\.js)"', page.content)
|
||||
assert asset is not None
|
||||
path = asset[1].decode()
|
||||
response = admin_client.get(path, **headers)
|
||||
assert response.status_code == 200
|
||||
assert "private" in response.headers["Cache-Control"]
|
||||
assert "max-age=" in response.headers["Cache-Control"]
|
||||
assert b"/admin/agent/opencode" in response.content
|
||||
repeated = admin_client.get(path, HTTP_IF_NONE_MATCH=response.headers["ETag"], **headers)
|
||||
assert repeated.status_code == 304
|
||||
assert repeated.content == b""
|
||||
assert repeated.headers["ETag"] == response.headers["ETag"]
|
||||
assert admin_client.get("/admin/agent/opencode/path", **headers).headers["Cache-Control"] == "no-store"
|
||||
|
||||
|
||||
def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
|
||||
response = admin_client.get(
|
||||
"/admin/agent/opencode/global/event",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.streaming
|
||||
assert response.is_async
|
||||
assert response.headers["X-Accel-Buffering"] == "no"
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
|
||||
|
||||
def test_opencode_proxy_sse_returns_headers_before_restart_finishes(admin_client, live_opencode):
|
||||
from archivebox.core.asgi import application
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
from django.conf import settings as django_settings
|
||||
|
||||
owned_process = runtime._PROCESS
|
||||
assert owned_process is not None
|
||||
session_cookie_name = django_settings.SESSION_COOKIE_NAME
|
||||
session_cookie = admin_client.cookies[session_cookie_name].value
|
||||
path = "/admin/agent/opencode/global/event"
|
||||
|
||||
async def request_event_stream():
|
||||
communicator = ApplicationCommunicator(
|
||||
application,
|
||||
{
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0"},
|
||||
"http_version": "1.1",
|
||||
"method": "GET",
|
||||
"scheme": "http",
|
||||
"path": path,
|
||||
"raw_path": path.encode(),
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"host", ADMIN_TEST_HOST.encode()),
|
||||
(b"cookie", f"{session_cookie_name}={session_cookie}".encode()),
|
||||
(b"sec-fetch-site", b"same-origin"),
|
||||
],
|
||||
"client": ("127.0.0.1", 12345),
|
||||
"server": ("127.0.0.1", 5797),
|
||||
},
|
||||
)
|
||||
runtime._PROCESS_LOCK.acquire()
|
||||
runtime._PROCESS = None
|
||||
try:
|
||||
await communicator.send_input({"type": "http.request", "body": b"", "more_body": False})
|
||||
response_start = await communicator.receive_output(timeout=2)
|
||||
assert response_start["type"] == "http.response.start"
|
||||
assert response_start["status"] == 200
|
||||
finally:
|
||||
try:
|
||||
await communicator.send_input({"type": "http.disconnect"})
|
||||
await communicator.wait(timeout=5)
|
||||
finally:
|
||||
runtime._PROCESS = owned_process
|
||||
runtime._PROCESS_LOCK.release()
|
||||
await asyncio.get_running_loop().shutdown_default_executor()
|
||||
|
||||
asyncio.run(request_event_stream())
|
||||
assert runtime._PROCESS is owned_process
|
||||
assert owned_process.poll() is None
|
||||
|
||||
|
||||
def test_opencode_invalid_state_does_not_break_archivebox(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
runtime._stop_owned_process()
|
||||
invalid_state = live_opencode.config.state_dir / "config"
|
||||
invalid_state.rename(live_opencode.config.state_dir / "saved-config")
|
||||
invalid_state.write_text("Preserve this file.")
|
||||
|
||||
wrapper = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert wrapper.status_code == 200
|
||||
for url in (
|
||||
runtime._project_route(live_opencode.config.data_dir),
|
||||
"/admin/agent/opencode/global/health",
|
||||
):
|
||||
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 503
|
||||
assert str(invalid_state).encode() not in response.content
|
||||
|
||||
stream = admin_client.get("/admin/agent/opencode/global/event", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert stream.status_code == 200
|
||||
|
||||
async def read_failure():
|
||||
return b"".join([chunk async for chunk in stream.streaming_content])
|
||||
|
||||
assert asyncio.run(read_failure()) == b'event: error\ndata: {"error":"OpenCode unavailable"}\n\n'
|
||||
|
||||
for url in ("/health/", "/add/", "/admin/core/snapshot/"):
|
||||
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200
|
||||
assert invalid_state.read_text() == "Preserve this file."
|
||||
@@ -1,191 +1,11 @@
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
"""Run plugin-owned host integration cases in the ArchiveBox CI matrix."""
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST
|
||||
|
||||
|
||||
from .test_opencode_agent_1 import (
|
||||
from abx_plugins.plugins.opencode.archivebox.agent_3_cases import (
|
||||
pytestmark as pytestmark,
|
||||
_free_port as _free_port,
|
||||
_reset_runtime_config as _reset_runtime_config,
|
||||
_set_archivebox_config as _set_archivebox_config,
|
||||
opencode_archive_config as opencode_archive_config,
|
||||
installed_opencode as installed_opencode,
|
||||
live_opencode as live_opencode,
|
||||
test_opencode_proxy_blocks_cross_origin_mutation as test_opencode_proxy_blocks_cross_origin_mutation,
|
||||
test_opencode_cold_agent_wrapper_returns_before_server_starts as test_opencode_cold_agent_wrapper_returns_before_server_starts,
|
||||
test_opencode_proxy_serves_real_project_and_session as test_opencode_proxy_serves_real_project_and_session,
|
||||
test_opencode_proxy_restarts_server_for_an_existing_agent_page as test_opencode_proxy_restarts_server_for_an_existing_agent_page,
|
||||
test_opencode_proxy_does_not_wait_for_recovery_lock as test_opencode_proxy_does_not_wait_for_recovery_lock,
|
||||
test_opencode_proxy_sse_delivers_first_event_immediately as test_opencode_proxy_sse_delivers_first_event_immediately,
|
||||
)
|
||||
|
||||
|
||||
def test_opencode_proxy_blocks_cross_origin_mutation(admin_client, db, live_opencode):
|
||||
response = admin_client.post(
|
||||
"/admin/agent/opencode/session",
|
||||
data=b"{}",
|
||||
content_type="application/json",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_ORIGIN="https://evil.example",
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_cold_agent_wrapper_returns_before_server_starts(admin_client, installed_opencode):
|
||||
import time
|
||||
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
assert not runtime._owned_process_running()
|
||||
started = time.monotonic()
|
||||
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert time.monotonic() - started < 3
|
||||
assert not runtime._owned_process_running()
|
||||
assert b'id="opencode-agent-welcome"' in response.content
|
||||
assert f'<iframe data-src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
|
||||
|
||||
|
||||
def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):
|
||||
workdir = str(live_opencode.config.data_dir.resolve())
|
||||
encoded_workdir = quote(workdir)
|
||||
|
||||
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert agent.status_code == 200
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
frame_path = runtime._project_route(live_opencode.config.data_dir)
|
||||
frame = admin_client.get(frame_path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert frame.status_code == 302
|
||||
session_id = frame.headers["Location"].rsplit("/", 1)[-1]
|
||||
|
||||
project = admin_client.get(
|
||||
f"/admin/agent/opencode/project/current?directory={encoded_workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert project.status_code == 200
|
||||
assert project.json()["id"] == "global"
|
||||
assert not project.json().get("vcs")
|
||||
|
||||
path = admin_client.get(
|
||||
f"/admin/agent/opencode/path?directory={encoded_workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert path.status_code == 200
|
||||
assert path.json()["directory"] == workdir
|
||||
|
||||
sessions = admin_client.get(
|
||||
f"/admin/agent/opencode/session?directory={encoded_workdir}&roots=true&limit=55",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert sessions.status_code == 200
|
||||
assert any(session["id"] == session_id and session["directory"] == workdir for session in sessions.json())
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
|
||||
|
||||
def test_opencode_proxy_restarts_server_for_an_existing_agent_page(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
old_process = runtime._PROCESS
|
||||
runtime._stop_owned_process()
|
||||
|
||||
response = admin_client.get(
|
||||
"/admin/agent/opencode/global/health",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert runtime._PROCESS is not None
|
||||
assert runtime._PROCESS is not old_process
|
||||
assert runtime._PROCESS.poll() is None
|
||||
|
||||
|
||||
def test_concurrent_opencode_startup_waits_until_server_is_ready(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
runtime._stop_owned_process()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(runtime._ensure_opencode, [live_opencode.settings] * 2))
|
||||
|
||||
assert results == [(True, ""), (True, "")]
|
||||
assert runtime._health(live_opencode.settings)
|
||||
|
||||
|
||||
def test_opencode_does_not_probe_or_replace_a_ready_owned_process(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
process = runtime._PROCESS
|
||||
settings = {**live_opencode.settings, "port": _free_port()}
|
||||
settings["origin"] = f"http://{settings['host']}:{settings['port']}"
|
||||
|
||||
ok, error = runtime._ensure_opencode(settings)
|
||||
|
||||
assert ok, error
|
||||
assert process is not None
|
||||
assert runtime._PROCESS is process
|
||||
assert process.poll() is None
|
||||
|
||||
|
||||
def test_opencode_proxy_does_not_wait_for_recovery_lock(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
workdir = quote(str(live_opencode.config.data_dir.resolve()))
|
||||
assert runtime._owned_process_ready()
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
runtime._PROCESS_LOCK.acquire()
|
||||
try:
|
||||
request = executor.submit(
|
||||
admin_client.get,
|
||||
f"/admin/agent/opencode/path?directory={workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
response = request.result(timeout=5)
|
||||
finally:
|
||||
runtime._PROCESS_LOCK.release()
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert str(live_opencode.config.data_dir.resolve()).encode() in response.content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["event", "global/event"])
|
||||
def test_opencode_proxy_sse_delivers_first_event_immediately(admin_client, live_opencode, path):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
# The real /event endpoint emits heartbeats indefinitely. A bounded
|
||||
# upstream read exposes accidental buffering without hanging the test.
|
||||
status, headers, body = runtime.proxy(
|
||||
{**live_opencode.settings, "timeout": 1},
|
||||
"GET",
|
||||
path,
|
||||
(),
|
||||
{},
|
||||
b"",
|
||||
)
|
||||
assert status == 200
|
||||
assert not isinstance(body, bytes)
|
||||
assert headers["Content-Type"] == "text/event-stream"
|
||||
asyncio.run(body.aclose())
|
||||
response = admin_client.get(
|
||||
f"/admin/agent/opencode/{path}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
async def first_event():
|
||||
stream = response.streaming_content
|
||||
try:
|
||||
async with asyncio.timeout(2):
|
||||
return await anext(stream)
|
||||
finally:
|
||||
await stream.aclose()
|
||||
|
||||
assert b"server.connected" in asyncio.run(first_event())
|
||||
@@ -1,291 +1,6 @@
|
||||
"""Exercise the real agent wrapper with Chromium's native storage failures."""
|
||||
"""Run plugin-owned host integration cases in the ArchiveBox CI matrix."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from .conftest import get_free_port, run_archivebox_cmd, start_archivebox_server, stop_archivebox_process
|
||||
from .test_opencode_agent_1 import _set_archivebox_config
|
||||
from .test_opencode_agent_1 import installed_opencode as installed_opencode
|
||||
from .test_opencode_agent_1 import live_opencode as live_opencode
|
||||
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
|
||||
from .test_server_security_browser import browser_runtime as browser_runtime
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent_server(installed_opencode, browser_runtime, request):
|
||||
port = get_free_port()
|
||||
mode, subdomains = getattr(request, "param", ("safe-onedomain-nojsreplay", False))
|
||||
base_url = f"http://archivebox.localhost:{port}" if subdomains else f"http://localhost:{port}"
|
||||
url = f"http://admin.archivebox.localhost:{port}" if subdomains else base_url
|
||||
config = installed_opencode.config
|
||||
_set_archivebox_config(
|
||||
config.data_dir,
|
||||
f"BASE_URL={base_url}",
|
||||
f"SERVER_SECURITY_MODE={mode}",
|
||||
)
|
||||
user = run_archivebox_cmd(
|
||||
[
|
||||
"shell",
|
||||
"-c",
|
||||
"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(username='agent-browser-test', password='test-password'); User.objects.create_user(username='agent-regular-test', password='test-password', is_staff=True)",
|
||||
],
|
||||
cwd=config.data_dir,
|
||||
env=config.env,
|
||||
)
|
||||
assert user.returncode == 0, user.stderr or user.stdout
|
||||
process = start_archivebox_server(config.data_dir, port=port, env=config.env, log_name="agent-browser-server.log")
|
||||
try:
|
||||
yield url, config.data_dir, process
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
stop_archivebox_process(process)
|
||||
|
||||
|
||||
def _login_cookie(server_url, username="agent-browser-test"):
|
||||
session = requests.Session()
|
||||
login_url = server_url + "/admin/login/"
|
||||
page = session.get(login_url, timeout=10)
|
||||
assert page.status_code == 200
|
||||
token = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', page.text)
|
||||
assert token is not None
|
||||
response = session.post(
|
||||
login_url,
|
||||
data={"username": username, "password": "test-password", "csrfmiddlewaretoken": token[1]},
|
||||
headers={"Referer": login_url},
|
||||
allow_redirects=False,
|
||||
timeout=10,
|
||||
)
|
||||
assert response.status_code == 302
|
||||
return "; ".join(f"{key}={value}" for key, value in session.cookies.items())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"agent_server",
|
||||
[
|
||||
("safe-onedomain-nojsreplay", False),
|
||||
("safe-subdomains-fullreplay", True),
|
||||
("danger-onedomain-fullreplay", False),
|
||||
("auto", False),
|
||||
("auto", True),
|
||||
],
|
||||
indirect=True,
|
||||
from abx_plugins.plugins.opencode.archivebox.browser_1_cases import (
|
||||
pytestmark as pytestmark,
|
||||
test_agent_navigation_stays_inside_mount as test_agent_navigation_stays_inside_mount,
|
||||
)
|
||||
def test_agent_navigation_stays_inside_mount(agent_server, browser_runtime):
|
||||
server_url, data_dir, _ = agent_server
|
||||
filename = "proxy acceptance #é.txt"
|
||||
(data_dir / filename).write_text("Proxy file read: café", encoding="utf-8")
|
||||
script = r"""
|
||||
const assert = require('node:assert/strict');
|
||||
const puppeteer = require('puppeteer');
|
||||
const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
|
||||
(async () => {
|
||||
const browser = await puppeteer.launch({executablePath: config.chrome, headless: true,
|
||||
args: ['--no-sandbox', '--disable-frame-rate-limit']});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
const escapedRequests = [];
|
||||
const agentRequests = [];
|
||||
page.on('request', request => {
|
||||
const url = new URL(request.url());
|
||||
if (url.pathname.startsWith('/admin/agent/opencode/')) agentRequests.push(url.pathname);
|
||||
if (request.frame()?.parentFrame() && url.origin === config.url &&
|
||||
!url.pathname.startsWith('/admin/agent/opencode/')) {
|
||||
escapedRequests.push(request.method() + ' ' + url.pathname);
|
||||
}
|
||||
});
|
||||
await page.setViewport({width: 1440, height: 900});
|
||||
await page.goto(config.url + '/admin/login/', {waitUntil: 'domcontentloaded'});
|
||||
await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
|
||||
await page.locator('#login-form input[name="password"]').fill('test-password');
|
||||
await Promise.all([page.waitForNavigation({waitUntil: 'domcontentloaded'}),
|
||||
page.locator('#login-form input[type="submit"]').click()]);
|
||||
assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'load'})).status(), 200);
|
||||
// The welcome dialog must not start a hidden OpenCode app: its cold startup
|
||||
// and asset load compete with rendering the very dialog the user must read.
|
||||
assert.deepEqual(agentRequests, [], 'OpenCode started before the user dismissed its welcome dialog');
|
||||
await page.locator('::-p-aria(Start using Agent[role="button"])').click();
|
||||
const frame = await (await page.waitForSelector('iframe')).contentFrame();
|
||||
await frame.waitForSelector('a[href*="/session"]');
|
||||
const links = await frame.$$eval('a[href*="/session"]', nodes => nodes.map(node => node.getAttribute('href')));
|
||||
assert.ok(links.length, 'OpenCode must expose session navigation');
|
||||
for (const href of links) assert.ok(href.startsWith('/admin/agent/opencode/'), href);
|
||||
const initialSessionUrl = frame.url();
|
||||
await frame.locator('::-p-aria(New session[role="button"])').click();
|
||||
await frame.waitForFunction(previous => location.href !== previous, {}, initialSessionUrl);
|
||||
await frame.waitForSelector('[contenteditable="true"]');
|
||||
let navigation;
|
||||
for (const link of await frame.$$('a[href*="/session"]')) {
|
||||
if (await link.isVisible() && await link.evaluate(node => node.href) !== frame.url()) {
|
||||
navigation = link;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert.ok(navigation, 'A visible link must navigate to another session route');
|
||||
await navigation.click();
|
||||
await frame.waitForSelector('[contenteditable="true"]');
|
||||
assert.ok(new URL(frame.url()).pathname.startsWith('/admin/agent/opencode/'), frame.url());
|
||||
const sessionUrl = frame.url();
|
||||
assert.equal((await frame.goto(sessionUrl, {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
await frame.waitForSelector('[contenteditable="true"]');
|
||||
assert.ok(!(await frame.$eval('body', node => node.innerText)).includes('Something went wrong'));
|
||||
// Real JSON methods, encoded file paths, and both SSE subscriptions must
|
||||
// work on the admin origin in every routing mode, without installing or
|
||||
// authenticating a paid model provider during the test.
|
||||
const requests = await frame.evaluate(async filename => {
|
||||
const base = location.origin + '/admin/agent/opencode';
|
||||
const json = async (path, method = 'GET', body) => {
|
||||
const response = await fetch(base + path, {
|
||||
method, headers: {'Content-Type': 'application/json'},
|
||||
...(body === undefined ? {} : {body: JSON.stringify(body)}),
|
||||
});
|
||||
if (!response.ok) throw new Error(method + ' ' + path + ': ' + response.status);
|
||||
if (!response.headers.get('Content-Type')?.includes('application/json')) {
|
||||
throw new Error('Non-JSON response: ' + path);
|
||||
}
|
||||
return response.json();
|
||||
};
|
||||
const providers = await json('/provider');
|
||||
const auth = await json('/provider/auth');
|
||||
const file = await json('/file/content?path=' + encodeURIComponent(filename));
|
||||
const session = await json('/session', 'POST', {title: 'Proxy request verification'});
|
||||
let renamed;
|
||||
try {
|
||||
renamed = await json('/session/' + session.id, 'PATCH', {title: 'Proxy: café & # symbols'});
|
||||
const readback = await json('/session/' + session.id);
|
||||
if (readback.title !== renamed.title) throw new Error('Session mutation was not persisted');
|
||||
} finally { await json('/session/' + session.id, 'DELETE'); }
|
||||
const streams = [];
|
||||
for (const path of ['/event', '/global/event']) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 5000);
|
||||
try {
|
||||
const response = await fetch(base + path, {signal: controller.signal});
|
||||
if (!response.ok || !response.headers.get('Content-Type')?.includes('text/event-stream')) {
|
||||
throw new Error('Invalid event stream: ' + path);
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const first = await reader.read();
|
||||
streams.push(new TextDecoder().decode(first.value));
|
||||
await reader.cancel();
|
||||
} finally { clearTimeout(timeout); controller.abort(); }
|
||||
}
|
||||
return {providers: providers.all.map(provider => provider.id), auth, file, renamed, streams};
|
||||
}, config.filename);
|
||||
for (const provider of ['openai', 'anthropic']) assert.ok(requests.providers.includes(provider), provider);
|
||||
assert.ok(requests.auth.openai.some(method => method.type === 'oauth'));
|
||||
assert.equal(requests.file.content, 'Proxy file read: café');
|
||||
assert.equal(requests.renamed.title, 'Proxy: café & # symbols');
|
||||
for (const event of requests.streams) assert.ok(event.includes('server.connected'), event);
|
||||
for (const provider of ['OpenAI', 'Anthropic']) {
|
||||
await frame.locator('[data-action="prompt-model"]').click();
|
||||
await frame.locator('::-p-aria(' + provider + '[role="button"])').click();
|
||||
if (provider === 'OpenAI') {
|
||||
await frame.waitForSelector('::-p-aria(ChatGPT Pro/Plus Headless[role="button"])');
|
||||
await frame.waitForSelector('::-p-aria(ChatGPT Pro/Plus Browser[role="button"])');
|
||||
await frame.locator('::-p-aria(API key Browser[role="button"])').click();
|
||||
}
|
||||
const input = await frame.waitForSelector('::-p-aria(' + provider + ' API key[role="textbox"])');
|
||||
assert.equal(await input.evaluate(node => node.value), '');
|
||||
await frame.waitForSelector('::-p-aria(Continue[role="button"])');
|
||||
await frame.locator('::-p-aria(Close[role="button"])').click();
|
||||
}
|
||||
// Submit a real shell command from a fresh draft. This exercises draft
|
||||
// promotion and the shell API without needing a paid model or API key.
|
||||
await frame.locator('::-p-aria(New session[role="button"])').click();
|
||||
await frame.waitForFunction(() => location.pathname.endsWith('/new-session'));
|
||||
await (await frame.waitForSelector('[contenteditable="true"]')).focus();
|
||||
const modifier = await page.evaluate(() => navigator.platform.includes('Mac') ? 'Meta' : 'Control');
|
||||
await page.keyboard.down(modifier);
|
||||
await page.keyboard.down('Shift');
|
||||
await page.keyboard.press('x');
|
||||
await page.keyboard.up('Shift');
|
||||
await page.keyboard.up(modifier);
|
||||
await page.keyboard.type('printf ABX_DRAFT_SHELL_OK');
|
||||
await frame.locator('::-p-aria(Send[role="button"])').click();
|
||||
await frame.waitForFunction(() => /\/session\/ses_/.test(location.pathname));
|
||||
assert.ok(new URL(frame.url()).pathname.startsWith('/admin/agent/opencode/'), frame.url());
|
||||
await frame.waitForFunction(async () => {
|
||||
const sessionID = location.pathname.match(/\/session\/(ses_[^/]+)/)[1];
|
||||
const response = await fetch('/admin/agent/opencode/session/' + sessionID + '/message');
|
||||
if (!response.ok) throw new Error('Shell message readback: ' + response.status);
|
||||
const messages = await response.json();
|
||||
return messages.some(message => message.parts.some(part =>
|
||||
part.type === 'tool' && part.state.status === 'completed' &&
|
||||
part.state.output.trim() === 'ABX_DRAFT_SHELL_OK'));
|
||||
}, {polling: 250});
|
||||
// The tabbed layout's own route parser must recognize mounted sessions.
|
||||
// Otherwise Home incorrectly appears selected and clicking it does nothing.
|
||||
const shellSessionUrl = frame.url();
|
||||
const home = await frame.waitForSelector('[aria-label="Home"]');
|
||||
assert.equal(await home.evaluate(node => node.getAttribute('aria-pressed')), 'false');
|
||||
await home.click();
|
||||
await frame.waitForFunction(() => location.pathname.replace(/\/+$/, '') === '/admin/agent/opencode');
|
||||
assert.equal(await home.evaluate(node => node.getAttribute('aria-pressed')), 'true');
|
||||
await home.click();
|
||||
await frame.waitForFunction(expected => location.href === expected, {}, shellSessionUrl);
|
||||
// Exercise the actual public PTY API and native browser WebSocket. No
|
||||
// intercepted traffic or replacement server: this runs a real shell.
|
||||
const terminal = await frame.evaluate(async () => {
|
||||
const base = location.origin + '/admin/agent/opencode';
|
||||
const created = await fetch(base + '/pty', {
|
||||
method: 'POST', headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({command: '/bin/sh', args: []}),
|
||||
});
|
||||
if (!created.ok) throw new Error('PTY create: ' + created.status);
|
||||
const pty = await created.json();
|
||||
try {
|
||||
const authorized = await fetch(base + '/pty/' + pty.id + '/connect-token', {
|
||||
method: 'POST', headers: {'x-opencode-ticket': '1'},
|
||||
});
|
||||
if (authorized.status !== 200) throw new Error('PTY ticket: ' + authorized.status);
|
||||
const {ticket} = await authorized.json();
|
||||
if (!ticket) throw new Error('PTY ticket missing');
|
||||
return await new Promise((resolve, reject) => {
|
||||
const url = new URL(base + '/pty/' + pty.id + '/connect');
|
||||
url.searchParams.set('ticket', ticket);
|
||||
url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const socket = new WebSocket(url);
|
||||
const timer = setTimeout(() => { socket.close(); reject(new Error('PTY output timed out')); }, 10000);
|
||||
let output = '';
|
||||
socket.onopen = () => socket.send("printf 'ABX_%s\\n' TERMINAL_OK\n");
|
||||
socket.onmessage = async event => {
|
||||
output += typeof event.data === 'string' ? event.data : await event.data.text();
|
||||
if (output.includes('ABX_TERMINAL_OK')) {
|
||||
clearTimeout(timer); socket.close(); resolve(output);
|
||||
}
|
||||
};
|
||||
socket.onerror = () => { clearTimeout(timer); reject(new Error('PTY WebSocket failed')); };
|
||||
});
|
||||
} finally { await fetch(base + '/pty/' + pty.id, {method: 'DELETE'}); }
|
||||
});
|
||||
assert.ok(terminal.includes('ABX_TERMINAL_OK'), terminal);
|
||||
assert.deepEqual(escapedRequests, [], 'OpenCode requests must stay inside its proxy mount');
|
||||
assert.equal((await page.goto(config.url + '/add/', {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
// A real click persists the dismissal: returning users load the Agent
|
||||
// directly, without another click or seeded browser preferences.
|
||||
assert.equal((await page.goto(config.url + '/admin/agent', {waitUntil: 'load'})).status(), 200);
|
||||
await page.waitForSelector('::-p-aria(Start using Agent[role="button"])', {hidden: true});
|
||||
const returningFrame = await (await page.waitForSelector('iframe[title="OpenCode Agent"]')).contentFrame();
|
||||
await returningFrame.waitForSelector('::-p-aria(New session[role="button"])');
|
||||
console.log('AGENT_NAVIGATION_OK');
|
||||
} finally { await browser.close(); }
|
||||
})().catch(error => { console.error(error); process.exitCode = 1; });
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[str(browser_runtime["node_binary"]), "-e", script],
|
||||
input=json.dumps({"chrome": str(browser_runtime["chrome_binary"]), "url": server_url, "filename": filename}),
|
||||
env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "AGENT_NAVIGATION_OK" in result.stdout
|
||||
@@ -1,90 +1,6 @@
|
||||
"""Exercise the real agent wrapper with Chromium's native storage failures."""
|
||||
"""Run plugin-owned host integration cases in the ArchiveBox CI matrix."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from .conftest import start_archivebox_server, stop_archivebox_process
|
||||
from .test_opencode_agent_1 import _set_archivebox_config
|
||||
from .test_opencode_agent_1 import installed_opencode as installed_opencode
|
||||
from .test_opencode_agent_1 import live_opencode as live_opencode
|
||||
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
|
||||
from .test_server_security_browser import browser_runtime as browser_runtime
|
||||
|
||||
|
||||
from .test_opencode_browser_1 import (
|
||||
from abx_plugins.plugins.opencode.archivebox.browser_2_cases import (
|
||||
pytestmark as pytestmark,
|
||||
agent_server as agent_server,
|
||||
_login_cookie as _login_cookie,
|
||||
test_agent_websocket_matches_http_security_policy as test_agent_websocket_matches_http_security_policy,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode,proxy_whitelist,use_cookie,http_status",
|
||||
[
|
||||
("unsafe-onedomain-noadmin", "", True, 403),
|
||||
("safe-onedomain-nojsreplay", "127.0.0.1/32", False, 200),
|
||||
("safe-onedomain-nojsreplay", "192.0.2.0/24", False, 302),
|
||||
],
|
||||
)
|
||||
def test_agent_websocket_matches_http_security_policy(agent_server, live_opencode, mode, proxy_whitelist, use_cookie, http_status):
|
||||
from urllib.parse import urlsplit
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import InvalidStatus
|
||||
|
||||
server_url, data_dir, process = agent_server
|
||||
cookie = _login_cookie(server_url)
|
||||
created = requests.post(
|
||||
server_url + "/admin/agent/opencode/pty",
|
||||
headers={"Cookie": cookie, "Origin": server_url},
|
||||
json={"command": "/bin/sh", "args": []},
|
||||
timeout=10,
|
||||
)
|
||||
assert created.status_code == 200
|
||||
pty_id = created.json()["id"]
|
||||
stop_archivebox_process(process)
|
||||
_set_archivebox_config(
|
||||
data_dir,
|
||||
f"SERVER_SECURITY_MODE={mode}",
|
||||
f"REVERSE_PROXY_WHITELIST={proxy_whitelist}",
|
||||
"REVERSE_PROXY_USER_HEADER=Remote-User",
|
||||
)
|
||||
restarted = start_archivebox_server(
|
||||
data_dir,
|
||||
port=urlsplit(server_url).port,
|
||||
env=live_opencode.config.env,
|
||||
log_name="agent-security-server.log",
|
||||
)
|
||||
headers = {"Cookie": cookie} if use_cookie else {"Remote-User": "agent-browser-test"}
|
||||
try:
|
||||
response = requests.get(server_url + "/admin/agent/opencode/global/health", headers=headers, allow_redirects=False, timeout=10)
|
||||
assert response.status_code == http_status
|
||||
|
||||
async def check_socket():
|
||||
connection = connect(
|
||||
server_url.replace("http", "ws", 1) + f"/admin/agent/opencode/pty/{pty_id}/connect",
|
||||
origin=server_url,
|
||||
additional_headers=headers,
|
||||
proxy=None,
|
||||
)
|
||||
if http_status != 200:
|
||||
with pytest.raises(InvalidStatus) as error:
|
||||
async with connection:
|
||||
pytest.fail("Disabled control plane or untrusted proxy accepted a WebSocket")
|
||||
assert error.value.response.status_code == 403
|
||||
return
|
||||
async with connection as socket, asyncio.timeout(10):
|
||||
await socket.send("printf 'ABX_%s\\n' PROXY_OK\n")
|
||||
output = ""
|
||||
while "ABX_PROXY_OK" not in output:
|
||||
chunk = await socket.recv()
|
||||
output += chunk.decode() if isinstance(chunk, bytes) else chunk
|
||||
assert "ABX_PROXY_OK" in output
|
||||
|
||||
asyncio.run(check_socket())
|
||||
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
|
||||
finally:
|
||||
deleted = requests.delete(live_opencode.settings["origin"] + f"/pty/{pty_id}", timeout=10)
|
||||
assert deleted.status_code == 200
|
||||
stop_archivebox_process(restarted)
|
||||
@@ -1,162 +1,7 @@
|
||||
"""Exercise the real agent wrapper with Chromium's native storage failures."""
|
||||
"""Run plugin-owned host integration cases in the ArchiveBox CI matrix."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from .test_opencode_agent_1 import installed_opencode as installed_opencode
|
||||
from .test_opencode_agent_1 import live_opencode as live_opencode
|
||||
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
|
||||
from .test_server_security_browser import browser_runtime as browser_runtime
|
||||
|
||||
|
||||
from .test_opencode_browser_1 import (
|
||||
from abx_plugins.plugins.opencode.archivebox.browser_3_cases import (
|
||||
pytestmark as pytestmark,
|
||||
agent_server as agent_server,
|
||||
_login_cookie as _login_cookie,
|
||||
test_agent_websocket_rejects_unauthorized_access as test_agent_websocket_rejects_unauthorized_access,
|
||||
test_agent_preserves_projects_and_survives_storage_failure as test_agent_preserves_projects_and_survives_storage_failure,
|
||||
)
|
||||
|
||||
|
||||
def test_agent_websocket_rejects_unauthorized_access(agent_server):
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import InvalidStatus
|
||||
|
||||
server_url, _, _ = agent_server
|
||||
admin_cookie = _login_cookie(server_url)
|
||||
regular_cookie = _login_cookie(server_url, "agent-regular-test")
|
||||
|
||||
async def check_denials():
|
||||
for path, cookie, origin in (
|
||||
("/admin/agent/opencode/pty/unknown/connect", "", server_url),
|
||||
("/admin/agent/opencode/pty/unknown/connect", regular_cookie, server_url),
|
||||
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, "https://untrusted.example"),
|
||||
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, None),
|
||||
("/health/", admin_cookie, server_url),
|
||||
):
|
||||
with pytest.raises(InvalidStatus) as error:
|
||||
async with connect(
|
||||
server_url.replace("http", "ws", 1) + path,
|
||||
origin=origin,
|
||||
additional_headers={"Cookie": cookie},
|
||||
proxy=None,
|
||||
):
|
||||
pytest.fail("Unauthorized WebSocket was accepted")
|
||||
assert error.value.response.status_code == 403
|
||||
|
||||
asyncio.run(check_denials())
|
||||
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
|
||||
|
||||
|
||||
def test_agent_preserves_projects_and_survives_storage_failure(agent_server, browser_runtime):
|
||||
server_url, data_dir, _ = agent_server
|
||||
script = r"""
|
||||
const assert = require('node:assert/strict');
|
||||
const puppeteer = require('puppeteer');
|
||||
const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
|
||||
(async () => {
|
||||
for (const disabled of [false, true]) {
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: config.chrome,
|
||||
headless: true,
|
||||
// Headless interaction must not depend on the desktop display clock.
|
||||
args: ['--no-sandbox', '--disable-frame-rate-limit', ...(disabled ? ['--disable-local-storage'] : [])],
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.goto(new URL('/admin/login/', config.url).href, {waitUntil: 'domcontentloaded'});
|
||||
await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
|
||||
await page.locator('#login-form input[name="password"]').fill('test-password');
|
||||
assert.equal(await page.$eval('#login-form input[name="username"]', element => element.value), 'agent-browser-test');
|
||||
assert.equal(await page.$eval('#login-form input[name="password"]', element => element.value), 'test-password');
|
||||
await Promise.all([
|
||||
page.waitForNavigation({waitUntil: 'domcontentloaded'}),
|
||||
page.locator('#login-form input[type="submit"]').click(),
|
||||
]);
|
||||
assert.ok(!page.url().includes('/admin/login/'), page.url());
|
||||
assert.equal((await page.goto(config.url, {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
const welcome = '#opencode-agent-welcome';
|
||||
await page.waitForSelector(welcome, {visible: true});
|
||||
if (disabled) {
|
||||
assert.equal(await page.evaluate(() => {
|
||||
try { return localStorage === null; }
|
||||
catch (error) { if (error.name !== 'SecurityError') throw error; return true; }
|
||||
}), true);
|
||||
} else {
|
||||
await page.click('#opencode-agent-welcome-dismiss');
|
||||
await page.waitForSelector(welcome, {hidden: true});
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
assert.equal(await page.$eval(welcome, element => element.hidden), true);
|
||||
const existing = {
|
||||
list: [{type: 'http', http: {url: 'http://other.example'}}],
|
||||
projects: {
|
||||
local: [{worktree: '/other-project', expanded: false}],
|
||||
'http://other.example': [{worktree: '/remote-project', expanded: true}],
|
||||
},
|
||||
lastProject: {'http://other.example': '/remote-project'},
|
||||
};
|
||||
await page.evaluate(value => localStorage.setItem('opencode.global.dat:server', JSON.stringify(value)), existing);
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
await page.waitForFunction(workdir => {
|
||||
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
|
||||
return state.projects.local?.some(project => project.worktree === workdir);
|
||||
}, {}, config.workdir);
|
||||
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('opencode.global.dat:server')));
|
||||
assert.deepEqual(saved.list, existing.list);
|
||||
assert.deepEqual(saved.projects['http://other.example'], existing.projects['http://other.example']);
|
||||
assert.equal(saved.lastProject['http://other.example'], '/remote-project');
|
||||
const projects = saved.projects.local;
|
||||
assert.ok(Array.isArray(projects), JSON.stringify({url: page.url(), saved, existing}));
|
||||
assert.deepEqual(projects.find(project => project.worktree === '/other-project'), {worktree: '/other-project', expanded: false});
|
||||
assert.equal(projects.filter(project => project.worktree === config.workdir).length, 1);
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
assert.equal(await page.evaluate(workdir => {
|
||||
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
|
||||
return state.projects.local.filter(project => project.worktree === workdir).length;
|
||||
}, config.workdir), 1);
|
||||
|
||||
// Fill the actual browser quota; do not replace or intercept Storage methods.
|
||||
const failure = await page.evaluate(() => {
|
||||
localStorage.clear();
|
||||
let low = 0, high = 8 * 1024 * 1024;
|
||||
while (low < high) {
|
||||
const size = Math.ceil((low + high) / 2);
|
||||
try { localStorage.setItem('padding', 'x'.repeat(size)); low = size; }
|
||||
catch (error) { if (error.name !== 'QuotaExceededError') throw error; high = size - 1; }
|
||||
}
|
||||
try { localStorage.setItem('extra', '1'); return null; }
|
||||
catch (error) { return error.name; }
|
||||
});
|
||||
assert.equal(failure, 'QuotaExceededError');
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
await page.waitForSelector(welcome, {visible: true});
|
||||
}
|
||||
await page.click('#opencode-agent-welcome-dismiss');
|
||||
await page.waitForSelector(welcome, {hidden: true});
|
||||
assert.equal((await page.goto(new URL('/add/', config.url).href, {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
console.log('AGENT_STORAGE_FAILURE_ISOLATED');
|
||||
})().catch(error => { console.error(error); process.exitCode = 1; });
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[str(browser_runtime["node_binary"]), "-e", script],
|
||||
input=json.dumps(
|
||||
{
|
||||
"chrome": str(browser_runtime["chrome_binary"]),
|
||||
"url": f"{server_url}/admin/agent",
|
||||
"workdir": str(data_dir.resolve()),
|
||||
},
|
||||
),
|
||||
env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "AGENT_STORAGE_FAILURE_ISOLATED" in result.stdout
|
||||
@@ -51,6 +51,8 @@ class TestProcessRuntimePaths:
|
||||
)
|
||||
|
||||
expected_dir = Path(process.pwd)
|
||||
assert process.status == Process.StatusChoices.EXITED
|
||||
assert process.exit_code == 0
|
||||
assert process.runtime_dir == expected_dir
|
||||
assert process.stdout_file == expected_dir / "stdout.log"
|
||||
assert process.stderr_file == expected_dir / "stderr.log"
|
||||
@@ -33,7 +33,7 @@ def run_plugin(name: str, snap: Path, **config: str):
|
||||
)
|
||||
|
||||
|
||||
def test_tlsnotary_preview_verifies_saved_evidence_without_remote_requests(snapshot, client, live_server, tmp_path):
|
||||
def test_tlsnotary_preview_verifies_saved_evidence_without_remote_requests(snapshot, client, live_server, tmp_path, browser_runtime):
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.core.routes_util import get_snapshot_host
|
||||
from archivebox.machine.models import Machine
|
||||
@@ -74,7 +74,10 @@ def test_tlsnotary_preview_verifies_saved_evidence_without_remote_requests(snaps
|
||||
hostname = host.split(":")[0]
|
||||
url = f"http://{hostname}:{port}{path}"
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=[f"--host-resolver-rules=MAP {hostname} 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=[f"--host-resolver-rules=MAP {hostname} 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
requests = []
|
||||
page.on("request", lambda request: requests.append(request.url))
|
||||
@@ -121,7 +124,7 @@ def test_tlsnotary_preview_verifies_saved_evidence_without_remote_requests(snaps
|
||||
browser.close()
|
||||
|
||||
|
||||
def test_opentimestamps_preview_reads_raw_proof_generation(snapshot, client, live_server, tmp_path):
|
||||
def test_opentimestamps_preview_reads_raw_proof_generation(snapshot, client, live_server, tmp_path, browser_runtime):
|
||||
import hashlib
|
||||
|
||||
from abx_plugins.plugins.base.testing import install_required_binary_from_config
|
||||
@@ -168,7 +171,10 @@ def test_opentimestamps_preview_reads_raw_proof_generation(snapshot, client, liv
|
||||
assert b"".join(raw.streaming_content) == (evidence / "hashes.json.ots").read_bytes()
|
||||
hostname = host.split(":")[0]
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=[f"--host-resolver-rules=MAP {hostname} 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=[f"--host-resolver-rules=MAP {hostname} 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
page.goto(f"http://{hostname}:{port}{path}")
|
||||
page.wait_for_function("document.querySelector('#manifest-sha256')?.textContent.length === 64")
|
||||
@@ -205,7 +211,7 @@ def test_opentimestamps_preview_reads_raw_proof_generation(snapshot, client, liv
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["safe-subdomains-fullreplay", "safe-onedomain-nojsreplay"])
|
||||
def test_legacy_proof_cards_and_viewers_on_canonical_snapshot_origin(snapshot, live_server, client, mode):
|
||||
def test_legacy_proof_cards_and_viewers_on_canonical_snapshot_origin(snapshot, live_server, client, mode, browser_runtime):
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.core.routes_util import get_snapshot_host
|
||||
from archivebox.machine.models import Machine
|
||||
@@ -256,6 +262,7 @@ def test_legacy_proof_cards_and_viewers_on_canonical_snapshot_origin(snapshot, l
|
||||
)
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1, MAP archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
|
||||
@@ -33,6 +33,7 @@ def run_add_until(args, env, condition, timeout=120):
|
||||
@pytest.mark.timeout(1200)
|
||||
def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
|
||||
initialized_archive,
|
||||
cached_abxpkg_lib_dir,
|
||||
free_tcp_port_factory,
|
||||
recursive_test_site,
|
||||
):
|
||||
@@ -59,8 +60,8 @@ def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
|
||||
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
|
||||
"CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
|
||||
"ABXPKG_LIB_DIR": str(cached_abxpkg_lib_dir),
|
||||
"CHROMEWEBSTORE_EXTENSIONS_DIR": str(cached_abxpkg_lib_dir / "chromewebstore/extensions"),
|
||||
"TIMEOUT": "90",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
|
||||
"SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
|
||||
|
||||
@@ -18,7 +18,11 @@ from .test_recursive_crawl_1 import (
|
||||
|
||||
|
||||
@pytest.mark.timeout(600)
|
||||
def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(initialized_archive, recursive_test_site):
|
||||
def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(
|
||||
initialized_archive,
|
||||
recursive_test_site,
|
||||
cached_abxpkg_lib_dir,
|
||||
):
|
||||
"""A real three-URL capture must not launch all Chrome hook trees at once."""
|
||||
urls = [recursive_test_site["root_url"], *recursive_test_site["child_urls"][:2]]
|
||||
env = os.environ.copy()
|
||||
@@ -27,7 +31,7 @@ def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(init
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
|
||||
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
|
||||
"ABXPKG_LIB_DIR": str(cached_abxpkg_lib_dir),
|
||||
"CHROME_HEADLESS": "true",
|
||||
"CHROME_SANDBOX": "false",
|
||||
"CHROME_ISOLATION": "crawl",
|
||||
|
||||
@@ -215,6 +215,8 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
|
||||
step for step in docker_workflow["jobs"]["build"]["steps"] if step.get("name") == "Validate exact built image version and commit"
|
||||
)
|
||||
assert version_check["if"] == "inputs.full_tests"
|
||||
assert "archivebox version" in version_check["run"]
|
||||
assert "Missing exact commit marker" in version_check["run"]
|
||||
pip_workflow = yaml.safe_load(PIP_WORKFLOW.read_text())
|
||||
install_script = next(
|
||||
step["run"] for step in pip_workflow["jobs"]["build"]["steps"] if step.get("name") == "Release wheel import and CLI smoke"
|
||||
@@ -238,11 +240,20 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
|
||||
assert 'echo "${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"' in tag_script
|
||||
assert 'echo "${DOCKERHUB_IMAGE}:${VERSION}"' in tag_script
|
||||
|
||||
docker_verify = next(step for step in docker_release["steps"] if step.get("name") == "Verify published Docker images run")
|
||||
docker_verify = next(
|
||||
step for step in docker_release["steps"] if step.get("name") == "Verify published tags contain the exact tested images"
|
||||
)
|
||||
verify_script = docker_verify["run"]
|
||||
assert '"${DOCKERHUB_IMAGE}:sha-${SHORT_SHA}"' in verify_script
|
||||
assert '"${GHCR_IMAGE}:sha-${SHORT_SHA}"' in verify_script
|
||||
assert '"${DOCKERHUB_IMAGE}:${VERSION}"' not in verify_script
|
||||
assert docker_verify["env"] == {
|
||||
"DOCKERHUB_TAGS": "${{ steps.docker_meta.outputs.dockerhub_tags }}",
|
||||
"GHCR_TAGS": "${{ steps.docker_meta.outputs.ghcr_tags }}",
|
||||
}
|
||||
assert '"${DOCKERHUB_IMAGE}@sha256:${digest}"' in verify_script
|
||||
assert 'imagetools create --dry-run "${REFS[@]}"' in verify_script
|
||||
assert 'imagetools inspect --raw "$image"' in verify_script
|
||||
assert 'sort == ["amd64", "arm64"]' in verify_script
|
||||
assert '[[ "$ACTUAL" == "$EXPECTED" ]]' in verify_script
|
||||
assert '"$DOCKERHUB_TAGS" "$GHCR_TAGS"' in verify_script
|
||||
|
||||
release_script = (REPO_ROOT / "bin" / "release.sh").read_text()
|
||||
assert "Never create GitHub Releases for automated rc builds" in release_script
|
||||
@@ -311,7 +322,7 @@ def test_stable_publication_requires_live_acceptance_before_upload():
|
||||
def test_staging_acceptance_uses_published_image_source_not_workflow_run_head():
|
||||
release = yaml.safe_load(RELEASE_WORKFLOW.read_text())
|
||||
docker_steps = release["jobs"]["docker-release"]["steps"]
|
||||
verified = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Verify published Docker images run")
|
||||
verified = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Verify published tags contain the exact tested images")
|
||||
source = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Record published image source")
|
||||
uploaded = next(i for i, step in enumerate(docker_steps) if step.get("name") == "Upload published image source")
|
||||
assert verified < source < uploaded
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from archivebox.services.resource_admission import ResourceAdmission, disk_backed_swap_free_bytes
|
||||
|
||||
@@ -55,6 +56,59 @@ def test_memory_observation_tracks_a_real_child_process():
|
||||
assert child.wait(timeout=10) == 0
|
||||
|
||||
|
||||
def test_memory_observation_counts_shared_pages_once_on_linux():
|
||||
admission = ResourceAdmission()
|
||||
before = admission.memory_headroom()
|
||||
assert before is not None
|
||||
size = 128 * 1024 * 1024
|
||||
child = subprocess.Popen(
|
||||
[
|
||||
"uv",
|
||||
"run",
|
||||
"python",
|
||||
"-c",
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
allocation = bytearray(128 * 1024 * 1024)
|
||||
children = []
|
||||
for _ in range(2):
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
sys.stdin.read()
|
||||
os._exit(0)
|
||||
children.append(pid)
|
||||
print(len(allocation), flush=True)
|
||||
sys.stdin.read()
|
||||
for pid in children:
|
||||
os.waitpid(pid, 0)
|
||||
""",
|
||||
],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
assert child.stdout is not None
|
||||
assert child.stdout.readline().strip() == str(size)
|
||||
during = admission.memory_headroom()
|
||||
assert during is not None
|
||||
growth = during[0] - before[0]
|
||||
assert growth > size // 2
|
||||
finite_cgroup = any(
|
||||
(path / "memory.max").is_file() and (path / "memory.max").read_text().strip() != "max" for path in admission.cgroup_paths()
|
||||
)
|
||||
if sys.platform.startswith("linux") and not finite_cgroup:
|
||||
# These three real processes share one allocation. Linux PSS and
|
||||
# summing RSS differ by two copies. A finite cgroup instead measures
|
||||
# the entire container, including unrelated concurrent test workers.
|
||||
assert growth < size * 2, f"Shared allocation counted repeatedly: {growth}"
|
||||
finally:
|
||||
assert child.stdin is not None
|
||||
child.stdin.close()
|
||||
assert child.wait(timeout=10) == 0
|
||||
|
||||
|
||||
def test_disk_backed_swap_headroom_excludes_zram_but_keeps_backing_swap():
|
||||
swaps = """Filename Type Size Used Priority
|
||||
/dev/zram0 partition 1000 100 100
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Replay actual recorded HTML at a different DPR, without a live-site fallback."""
|
||||
|
||||
import os
|
||||
import gzip
|
||||
import hashlib
|
||||
import zlib
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import pytest
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
from .conftest import localhost_session
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_archived_srcset_recovers_only_missing_image_candidates(snapshot, live_server, browser_runtime):
|
||||
from abx_plugins import get_plugins_dir
|
||||
from abx_plugins.plugins.base.testing import install_required_binary_from_config
|
||||
from archivebox.core.routes_util import get_snapshot_host
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
install_required_binary_from_config(Path(get_plugins_dir()) / "archivewebpage", "archivewebpage", env=dict(os.environ))
|
||||
port = urlsplit(live_server.url).port
|
||||
machine = Machine.current()
|
||||
machine.config = {
|
||||
**machine.config,
|
||||
"BASE_URL": f"http://archivebox.localhost:{port}",
|
||||
"SERVER_SECURITY_MODE": "safe-subdomains-fullreplay",
|
||||
}
|
||||
machine.save(update_fields=["config"])
|
||||
snapshot.url = "https://drive.google.com/drive/folders/1KpLl_1tcK0eeehzN980zbG-3M2nhbVks"
|
||||
snapshot.permissions = "public"
|
||||
snapshot.save(update_fields=["url", "permissions"])
|
||||
fixture = Path(__file__).parent / "fixtures/responsive-images/google-drive.wacz"
|
||||
(snapshot.output_dir / "archivewebpage").mkdir(parents=True)
|
||||
with zipfile.ZipFile(fixture) as archive:
|
||||
recorded_responses = archive.read("archive/data.warc.gz")
|
||||
host = get_snapshot_host(str(snapshot.id)).split(":")[0]
|
||||
origin = f"http://{host}:{port}"
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=[*browser_runtime["chrome_args"], f"--host-resolver-rules=MAP {host} 127.0.0.1"],
|
||||
)
|
||||
for density, archive_format, include_logo in (
|
||||
(1, "warc", True),
|
||||
(2, "warc", True),
|
||||
(2, "warc", False),
|
||||
(1, "wacz", True),
|
||||
(2, "wacz", True),
|
||||
):
|
||||
# zlib reads the first gzip member only: the same unmodified HTML
|
||||
# record, without the image record. No candidate should then work.
|
||||
recorded = gzip.decompress(recorded_responses) if include_logo else zlib.decompress(recorded_responses, wbits=31)
|
||||
if archive_format == "wacz":
|
||||
recorded = fixture.read_bytes()
|
||||
output = snapshot.output_dir / f"archivewebpage/google-drive.{archive_format}"
|
||||
output.write_bytes(recorded)
|
||||
context = browser.new_context(device_scale_factor=density)
|
||||
page = context.new_page()
|
||||
requests, responses = [], []
|
||||
context.on("request", lambda request: requests.append(request.url))
|
||||
page.on("response", lambda response: responses.append(response))
|
||||
page.goto(f"{origin}/archivewebpage/google-drive.{archive_format}?preview=1", wait_until="domcontentloaded")
|
||||
page.wait_for_function("window.archiveboxReplayReady")
|
||||
frame = next(frame for frame in page.frames if "/https://drive.google.com/drive/folders/" in frame.url)
|
||||
logo = frame.locator('img[srcset*="logo_drive_2026_color"]')
|
||||
logo.wait_for()
|
||||
decoded = logo.evaluate("async img => {try {await img.decode(); return img.naturalWidth > 0;} catch {return false;}}")
|
||||
assert decoded is include_logo
|
||||
images = [
|
||||
response for response in responses if "logo_drive_2026_color" in response.url and response.request.resource_type == "image"
|
||||
]
|
||||
assert images, [(response.url, response.status) for response in responses]
|
||||
rendered = images[-1]
|
||||
assert rendered.status == (200 if include_logo else 404)
|
||||
if include_logo:
|
||||
assert hashlib.sha256(rendered.body()).hexdigest() == "c7469a9f91bd4d5a7f0f85e7e346a01d5962ee0c15b484fac90c197029cb9261"
|
||||
if density == 2 and include_logo:
|
||||
assert "2x_web_48dp.png" in rendered.url
|
||||
assert rendered.headers["x-archivebox-image-fallback"].endswith("1x_web_48dp.png")
|
||||
# The repair applies to image loads only, not arbitrary requests
|
||||
# for a missing URL, and never changes the archived bytes.
|
||||
assert frame.evaluate("url => fetch(url).then(r => r.status)", rendered.url) == 404
|
||||
else:
|
||||
assert "x-archivebox-image-fallback" not in rendered.headers
|
||||
if density == 1:
|
||||
assert "1x_web_48dp.png" in rendered.url
|
||||
assert all(url.startswith((origin, "data:", "blob:")) for url in requests), requests
|
||||
with localhost_session() as session:
|
||||
raw = session.get(f"{origin}/archivewebpage/google-drive.{archive_format}?raw=1")
|
||||
assert raw.status_code == 200
|
||||
assert raw.content == recorded
|
||||
context.close()
|
||||
browser.close()
|
||||
@@ -269,3 +269,86 @@ def test_image_rewrite_uses_optional_saved_artifacts(tmp_path: Path):
|
||||
(responses / "index.jsonl").unlink()
|
||||
(responses / "index.jsonl").mkdir()
|
||||
assert rewrite() == (source, 0)
|
||||
|
||||
|
||||
def test_zip_preview_is_generic_and_raw_range_stays_seekable(tmp_path):
|
||||
import zipfile
|
||||
|
||||
output = tmp_path / "arbitrary-plugin" / "nested" / "bundle.ZIP"
|
||||
output.parent.mkdir(parents=True)
|
||||
with zipfile.ZipFile(output, "w") as archive:
|
||||
archive.writestr("inside/note.txt", "generic archive browsing")
|
||||
path = output.relative_to(tmp_path).as_posix()
|
||||
factory = RequestFactory()
|
||||
preview = serve_static_with_byterange_support(factory.get(f"/{path}?preview=1"), path, document_root=tmp_path)
|
||||
assert preview.status_code == 200
|
||||
assert preview["Content-Type"].startswith("text/html")
|
||||
assert b"indexZip" in preview.content
|
||||
assert b"bundle.ZIP" in preview.content
|
||||
assert b"raw=1" in preview.content
|
||||
assert b"downloads.json" not in preview.content
|
||||
raw = serve_static_with_byterange_support(factory.get(f"/{path}?raw=1", HTTP_RANGE="bytes=0-3"), path, document_root=tmp_path)
|
||||
assert raw.status_code == 206
|
||||
assert b"".join(raw.streaming_content) == b"PK\x03\x04"
|
||||
listing = serve_static_with_byterange_support(
|
||||
factory.get("/arbitrary-plugin/nested/?files=1"),
|
||||
"arbitrary-plugin/nested/",
|
||||
document_root=tmp_path,
|
||||
show_indexes=True,
|
||||
)
|
||||
assert b"bundle.ZIP?preview=1" in listing.content
|
||||
|
||||
|
||||
def test_directory_index_has_filter_and_per_entry_downloads(tmp_path: Path):
|
||||
(tmp_path / "nested").mkdir()
|
||||
(tmp_path / "note.txt").write_text("searchable text")
|
||||
response = serve_static_with_byterange_support(
|
||||
RequestFactory().get("/snapshot/demo/?files=1"),
|
||||
"",
|
||||
document_root=tmp_path,
|
||||
show_indexes=True,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
html = response.content.decode()
|
||||
assert 'aria-label="Filter files"' in html
|
||||
assert 'href="nested/?files=1&download=zip"' in html
|
||||
assert 'href="note.txt?raw=1" download' in html
|
||||
|
||||
|
||||
def test_casefold_paths_remain_inside_archive_root(tmp_path: Path):
|
||||
from django.core.exceptions import SuspiciousFileOperation
|
||||
from archivebox.misc.serve_static import _resolve_archive_path
|
||||
|
||||
(tmp_path / "Documents").mkdir()
|
||||
saved = tmp_path / "Documents" / "Note.txt"
|
||||
saved.write_text("inside root")
|
||||
resolved, relative = _resolve_archive_path(tmp_path, "documents/note.TXT")
|
||||
assert resolved.read_text() == "inside root"
|
||||
assert resolved.is_relative_to(tmp_path)
|
||||
assert relative.casefold() == "documents/note.txt"
|
||||
with pytest.raises(SuspiciousFileOperation):
|
||||
_resolve_archive_path(tmp_path, "../DOCUMENTS/note.TXT")
|
||||
|
||||
|
||||
def test_directory_previews_only_small_supported_files(tmp_path: Path):
|
||||
import sqlite3
|
||||
|
||||
with sqlite3.connect(tmp_path / "database.sqlite3") as database:
|
||||
database.execute("CREATE TABLE example (value TEXT)")
|
||||
(tmp_path / "small.txt").write_text("small preview")
|
||||
(tmp_path / "exact-limit.txt").write_bytes(b"x" * (100 * 1024))
|
||||
(tmp_path / "unknown.bin").write_bytes(b"\x00\x01")
|
||||
(tmp_path / "picture.svg").write_text('<svg xmlns="http://www.w3.org/2000/svg"/>')
|
||||
response = serve_static_with_byterange_support(
|
||||
RequestFactory().get("/snapshot/demo/?files=1"),
|
||||
"",
|
||||
document_root=tmp_path,
|
||||
show_indexes=True,
|
||||
is_archive_replay=True,
|
||||
)
|
||||
html = response.content.decode()
|
||||
assert 'data-url="small.txt?raw=1"' in html
|
||||
assert 'src="picture.svg?raw=1"' in html
|
||||
assert 'data-url="exact-limit.txt?raw=1"' not in html
|
||||
assert 'data-url="unknown.bin?raw=1"' not in html
|
||||
assert 'data-url="database.sqlite3?raw=1"' not in html
|
||||
@@ -15,7 +15,7 @@ import pytest
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
from .conftest import resolve_abxpkg_chrome_env, run_python_cwd
|
||||
from .conftest import run_python_cwd
|
||||
from .conftest import (
|
||||
cli_env,
|
||||
get_free_port,
|
||||
@@ -385,35 +385,6 @@ main().catch(async (error) => {
|
||||
"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def browser_runtime(initialized_archive: Path):
|
||||
shared_lib = initialized_archive / "lib"
|
||||
env = cli_env(
|
||||
ABXPKG_LIB_DIR=str(shared_lib),
|
||||
CHROME_HEADLESS="True",
|
||||
CHROME_SANDBOX="False",
|
||||
CHROME_ISOLATION="snapshot",
|
||||
)
|
||||
env.pop("CHROME_BINARY", None)
|
||||
install_result = run_archivebox_cmd(
|
||||
["install", "chrome"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=900,
|
||||
)
|
||||
assert install_result.returncode == 0, install_result.stderr or install_result.stdout
|
||||
|
||||
resolved_env = resolve_abxpkg_chrome_env(shared_lib, env)
|
||||
|
||||
return {
|
||||
"lib_dir": shared_lib,
|
||||
"node_modules_dir": Path(resolved_env["NODE_MODULES_DIR"]),
|
||||
"node_path": resolved_env["NODE_PATH"],
|
||||
"node_binary": Path(resolved_env["NODE_BINARY"]),
|
||||
"chrome_binary": Path(resolved_env["CHROME_BINARY"]),
|
||||
}
|
||||
|
||||
|
||||
def _seed_archive(data_dir: Path) -> dict[str, object]:
|
||||
script = textwrap.dedent(
|
||||
"""
|
||||
|
||||
@@ -37,8 +37,10 @@ def test_foreground_shutdown_second_signal_exits_immediately():
|
||||
from archivebox.core.shutdown_util import foreground_shutdown_signals
|
||||
|
||||
with foreground_shutdown_signals(first_signal_message=None) as state:
|
||||
print("READY", flush=True)
|
||||
try:
|
||||
# The sender may signal as soon as READY is flushed, before
|
||||
# signal.pause(); the handler must already be inside this try.
|
||||
print("READY", flush=True)
|
||||
signal.pause()
|
||||
except KeyboardInterrupt:
|
||||
print(f"FIRST:{state.signal_name}", flush=True)
|
||||
|
||||
@@ -313,7 +313,7 @@ def test_responses_card_uses_relative_preview_urls_in_static_export(snapshot):
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.parametrize("status", ["queued", "started", "paused"])
|
||||
def test_empty_snapshot_keeps_live_progress_until_files_are_opened(snapshot, live_server, status):
|
||||
def test_empty_snapshot_keeps_live_progress_until_files_are_opened(snapshot, live_server, status, browser_runtime):
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
@@ -329,7 +329,10 @@ def test_empty_snapshot_keeps_live_progress_until_files_are_opened(snapshot, liv
|
||||
snapshot.save(update_fields=["permissions", "status"])
|
||||
Path(snapshot.output_dir).mkdir(parents=True, exist_ok=True)
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
page.goto(f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}")
|
||||
assert page.locator("#progress-monitor").is_visible()
|
||||
@@ -344,7 +347,7 @@ def test_empty_snapshot_keeps_live_progress_until_files_are_opened(snapshot, liv
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_opening_files_from_queued_snapshot_stops_detached_progress_monitor(snapshot, live_server):
|
||||
def test_opening_files_from_queued_snapshot_stops_detached_progress_monitor(snapshot, live_server, browser_runtime):
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
@@ -362,7 +365,10 @@ def test_opening_files_from_queued_snapshot_stops_detached_progress_monitor(snap
|
||||
errors = []
|
||||
progress_requests = []
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
page.on("pageerror", lambda error: errors.append(str(error)))
|
||||
page.on("console", lambda message: errors.append(message.text) if message.type == "error" else None)
|
||||
@@ -381,7 +387,7 @@ def test_opening_files_from_queued_snapshot_stops_detached_progress_monitor(snap
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_stack_cover_and_expanded_card_load_same_document(snapshot, live_server):
|
||||
def test_stack_cover_and_expanded_card_load_same_document(snapshot, live_server, browser_runtime):
|
||||
from urllib.parse import urlsplit
|
||||
from playwright.sync_api import sync_playwright
|
||||
from archivebox.machine.models import Machine
|
||||
@@ -394,7 +400,10 @@ def test_stack_cover_and_expanded_card_load_same_document(snapshot, live_server)
|
||||
snapshot.save(update_fields=["permissions"])
|
||||
save_output(snapshot, "defuddle")
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page(viewport={"width": 390, "height": 844})
|
||||
page.goto(f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}/index.html")
|
||||
assert "{#" not in page.locator("body").inner_text()
|
||||
@@ -422,7 +431,7 @@ def test_stack_cover_and_expanded_card_load_same_document(snapshot, live_server)
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_forumdl_card_and_full_view_render_saved_thread(snapshot, live_server):
|
||||
def test_forumdl_card_and_full_view_render_saved_thread(snapshot, live_server, browser_runtime):
|
||||
from urllib.parse import urlsplit
|
||||
from playwright.sync_api import sync_playwright
|
||||
from archivebox.core.routes_util import get_snapshot_host
|
||||
@@ -451,7 +460,10 @@ def test_forumdl_card_and_full_view_render_saved_thread(snapshot, live_server):
|
||||
viewer_url = f"http://{host}:{port}/forumdl/forum.jsonl?preview=1"
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=[f"--host-resolver-rules=MAP {host} 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=[f"--host-resolver-rules=MAP {host} 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
page.goto(viewer_url)
|
||||
page.locator(".thread-title").wait_for()
|
||||
@@ -481,7 +493,7 @@ def test_forumdl_card_and_full_view_render_saved_thread(snapshot, live_server):
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_pdf_card_shows_pdf_fallback_without_any_preview_image(snapshot, live_server):
|
||||
def test_pdf_card_shows_pdf_fallback_without_any_preview_image(snapshot, live_server, browser_runtime):
|
||||
from urllib.parse import urlsplit
|
||||
from playwright.sync_api import sync_playwright
|
||||
from archivebox.core.routes_util import get_snapshot_host
|
||||
@@ -508,7 +520,10 @@ def test_pdf_card_shows_pdf_fallback_without_any_preview_image(snapshot, live_se
|
||||
host = get_snapshot_host(str(snapshot.id)).split(":")[0]
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=[f"--host-resolver-rules=MAP {host} 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=[f"--host-resolver-rules=MAP {host} 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page(viewport={"width": 250, "height": 170})
|
||||
page.set_content("<h1>Archived page</h1>")
|
||||
output.write_bytes(page.pdf())
|
||||
@@ -522,7 +537,7 @@ def test_pdf_card_shows_pdf_fallback_without_any_preview_image(snapshot, live_se
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_responsive_header_and_expanded_stack_keep_full_view_in_page_flow(snapshot, live_server, tmp_path):
|
||||
def test_responsive_header_and_expanded_stack_keep_full_view_in_page_flow(snapshot, live_server, tmp_path, browser_runtime):
|
||||
import zipfile
|
||||
from urllib.parse import urlsplit
|
||||
from playwright.sync_api import sync_playwright
|
||||
@@ -545,7 +560,10 @@ def test_responsive_header_and_expanded_stack_keep_full_view_in_page_flow(snapsh
|
||||
expected_size = snapshot.get_html_details_context()["size"]
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
page.goto(f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}/index.html", wait_until="domcontentloaded")
|
||||
actions = page.locator(".header-url-actions")
|
||||
@@ -675,13 +693,16 @@ def test_responsive_header_and_expanded_stack_keep_full_view_in_page_flow(snapsh
|
||||
browser.close()
|
||||
|
||||
|
||||
def test_ios_phone_header_keeps_actions_and_capture_date_on_one_row(snapshot, live_server):
|
||||
def test_ios_phone_header_keeps_actions_and_capture_date_on_one_row(snapshot, live_server, browser_runtime):
|
||||
from urllib.parse import urlsplit
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
port = urlsplit(live_server.url).port
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page(viewport={"width": 390, "height": 844})
|
||||
page.goto(f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}/index.html", wait_until="domcontentloaded")
|
||||
for native_app in (False, True):
|
||||
@@ -703,7 +724,7 @@ def test_ios_phone_header_keeps_actions_and_capture_date_on_one_row(snapshot, li
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_year_badges_attach_selected_capture_to_its_year(snapshot, live_server, tmp_path):
|
||||
def test_year_badges_attach_selected_capture_to_its_year(snapshot, live_server, tmp_path, browser_runtime):
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlsplit
|
||||
from playwright.sync_api import sync_playwright
|
||||
@@ -726,7 +747,10 @@ def test_year_badges_attach_selected_capture_to_its_year(snapshot, live_server,
|
||||
save_output(copy, "defuddle")
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
for selected, expected_year, expected_date in ((snapshot, "2026", "2026-09-23"), (copies[1], "2025", "2025-03-02")):
|
||||
page.goto(f"http://web.archivebox.localhost:{port}{selected.get_absolute_url()}/index.html", wait_until="domcontentloaded")
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import json
|
||||
from abx_plugins.plugins.opencode.archivebox.host_cases import (
|
||||
test_add_view_hides_agent_link_when_opencode_is_disabled as test_add_view_hides_agent_link_when_opencode_is_disabled,
|
||||
)
|
||||
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
@@ -130,19 +134,6 @@ def test_add_view_admin_renders_plugin_config_grid(client, admin_user):
|
||||
assert b"plugin_config__wget__WGET_ENABLED" not in response.content
|
||||
|
||||
|
||||
def test_add_view_hides_agent_link_when_opencode_is_disabled(client, admin_user):
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
|
||||
client.force_login(admin_user)
|
||||
|
||||
response = client.get(reverse("add"), HTTP_HOST=ADMIN_HOST)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert b"/admin/agent" not in response.content
|
||||
assert b"Crawl with AI" not in response.content
|
||||
|
||||
|
||||
def test_add_view_staff_user_cannot_override_raw_or_plugin_config(client):
|
||||
staff_user = User.objects.create_user(
|
||||
username="addviewstaff",
|
||||
@@ -668,6 +659,14 @@ def test_plugin_grid_includes_all_visible_plugins_and_timestamps_last(admin_clie
|
||||
assert len(names) == len(expected)
|
||||
for name in expected:
|
||||
assert f'data-plugin-name="{name}"'.encode() in response.content
|
||||
# The same schema drives both editors, including newly installed plugins.
|
||||
# Check every plugin's category and rendered controls, not a fixed allowlist.
|
||||
catalog = get_plugin_catalog()
|
||||
for group in form.plugin_groups:
|
||||
for card in group["plugins"]:
|
||||
assert group["field_name"] == f"{catalog[card['name']].config.category}_plugins"
|
||||
for field in card["config_fields"]:
|
||||
assert f'name="{field["input_name"]}"'.encode() in response.content
|
||||
postprocessing = next(group for group in form.plugin_groups if group["field_name"] == "postprocessing_plugins")
|
||||
assert postprocessing["plugins"][-1]["name"] == "opentimestamps"
|
||||
assert b'name="plugin_config__opentimestamps__OPENTIMESTAMPS_TIMEOUT"' in response.content
|
||||
|
||||
@@ -10,6 +10,10 @@ from django.test import RequestFactory
|
||||
from django.urls import reverse
|
||||
import html
|
||||
from uuid import uuid4
|
||||
from abx_plugins.plugins.opencode.archivebox.host_cases import (
|
||||
test_admin_navigation_hides_agent_link_when_opencode_is_disabled as test_admin_navigation_hides_agent_link_when_opencode_is_disabled,
|
||||
)
|
||||
|
||||
|
||||
from archivebox.tests.conftest import cli_env, run_archivebox_cmd
|
||||
from archivebox.tests.conftest import install_real_binary
|
||||
@@ -385,19 +389,6 @@ def test_snapshot_file_icons_link_to_migrated_root_outputs_and_show_all_plugins(
|
||||
assert f'href="{result_admin.get_output_files_url(root_result)}" data-no-preview="1" title="Open output folder"' in cards_html
|
||||
|
||||
|
||||
def test_admin_navigation_hides_agent_link_when_opencode_is_disabled(client, admin_user):
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
|
||||
client.force_login(admin_user)
|
||||
|
||||
response = client.get(reverse("admin:index"), HTTP_HOST="admin.archivebox.localhost:5797")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert b"/admin/agent" not in response.content
|
||||
assert b">\xf0\x9f\x92\xac AI<" not in response.content
|
||||
|
||||
|
||||
def test_archiveresult_admin_zip_links():
|
||||
from archivebox.core.admin_archiveresults import ArchiveResultAdmin
|
||||
from archivebox.core.models import ArchiveResult
|
||||
|
||||
@@ -119,6 +119,7 @@ def test_isolated_snapshot_delete_button_hands_off_to_admin_without_sharing_admi
|
||||
admin_user,
|
||||
real_hash_projection,
|
||||
live_server,
|
||||
browser_runtime,
|
||||
):
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
@@ -146,6 +147,7 @@ def test_isolated_snapshot_delete_button_hands_off_to_admin_without_sharing_admi
|
||||
snap_origin = f"http://{get_snapshot_host(str(snapshot.id))}"
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(
|
||||
executable_path=str(browser_runtime["chrome_binary"]),
|
||||
args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1, MAP archivebox.localhost 127.0.0.1"],
|
||||
)
|
||||
context = browser.new_context()
|
||||
|
||||
@@ -18,6 +18,8 @@ from archivebox.tests.conftest import (
|
||||
start_archivebox_server,
|
||||
stop_server,
|
||||
get_http_response,
|
||||
localhost_session,
|
||||
wait_for_snapshot_capture,
|
||||
)
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
@@ -25,7 +27,7 @@ pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def _login_admin_over_full_server(port: int) -> tuple[requests.Session, str]:
|
||||
session = requests.Session()
|
||||
session = localhost_session()
|
||||
get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
|
||||
login_page = session.get(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
@@ -58,27 +60,13 @@ def _login_admin_over_full_server(port: int) -> tuple[requests.Session, str]:
|
||||
return session, add_csrf_match.group(1)
|
||||
|
||||
|
||||
def _stop_runner_worker(data_dir) -> None:
|
||||
script = """
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, stop_worker
|
||||
supervisor = get_existing_supervisord_process()
|
||||
assert supervisor is not None
|
||||
stop_worker(supervisor, "worker_runner")
|
||||
print("stopped")
|
||||
"""
|
||||
result = run_archivebox_cmd(["manage", "shell", "-c", script], cwd=data_dir, timeout=60)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
|
||||
def _create_private_snapshot_over_full_server(
|
||||
data_dir,
|
||||
session: requests.Session,
|
||||
port: int,
|
||||
csrf_token: str,
|
||||
url: str,
|
||||
env: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
_stop_runner_worker(data_dir)
|
||||
response = session.post(
|
||||
f"http://admin.archivebox.localhost:{port}/add/",
|
||||
headers={"Referer": f"http://admin.archivebox.localhost:{port}/add/"},
|
||||
@@ -107,15 +95,9 @@ def _create_private_snapshot_over_full_server(
|
||||
)
|
||||
assert response.status_code in (302, 303), response.text
|
||||
|
||||
with use_archivebox_db(data_dir):
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
crawl = Crawl.objects.order_by("-created_at").get()
|
||||
|
||||
stop_server(data_dir)
|
||||
run_result = run_archivebox_cmd(["run", f"--crawl-id={crawl.id}"], cwd=data_dir, timeout=120, env=env)
|
||||
assert run_result.returncode == 0, run_result.stderr or run_result.stdout
|
||||
start_archivebox_server(data_dir, env=env, port=port)
|
||||
# Let the running server consume its own /add/ submission, as it does for
|
||||
# users. Manually driving a stopped runner would hide queue/lifecycle bugs.
|
||||
wait_for_snapshot_capture(data_dir, url, timeout=120)
|
||||
|
||||
with use_archivebox_db(data_dir):
|
||||
from archivebox.core.models import Snapshot
|
||||
@@ -327,8 +309,7 @@ class TestPublicIndex:
|
||||
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
session = localhost_session()
|
||||
response = session.get(
|
||||
f"http://archivebox.localhost:{port}/",
|
||||
timeout=10,
|
||||
@@ -729,7 +710,6 @@ class TestPublicIndex:
|
||||
port,
|
||||
csrf_token,
|
||||
recursive_test_site["root_url"],
|
||||
env,
|
||||
)
|
||||
snap_url = f"http://{snapshot['host']}/index.html"
|
||||
|
||||
@@ -775,7 +755,6 @@ class TestPublicIndex:
|
||||
port,
|
||||
csrf_token,
|
||||
recursive_test_site["root_url"],
|
||||
env,
|
||||
)
|
||||
snap_url = f"http://{snapshot['host']}/index.html"
|
||||
|
||||
|
||||
@@ -1,10 +1,29 @@
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from threading import Thread
|
||||
from datetime import datetime, timezone
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.misc.util import download_url, find_all_urls, fix_url_from_markdown, parse_date, sanitize_extracted_url
|
||||
from archivebox.tests.conftest import get_free_port, run_python_cwd
|
||||
|
||||
|
||||
def test_port_allocations_do_not_repeat_across_processes(tmp_path):
|
||||
script = """
|
||||
import json
|
||||
from archivebox.tests.conftest import get_free_port
|
||||
print(json.dumps([get_free_port() for _ in range(256)]))
|
||||
"""
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
results = list(pool.map(lambda _: run_python_cwd(script, tmp_path), range(4)))
|
||||
ports = [get_free_port() for _ in range(256)]
|
||||
for stdout, stderr, returncode in results:
|
||||
assert returncode == 0, stderr
|
||||
ports.extend(json.loads(stdout))
|
||||
assert len(ports) == 1280
|
||||
assert len(set(ports)) == len(ports), "Parallel services must not receive the same port"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""ZIP outputs use the generic static-file explorer and bounded range reads."""
|
||||
|
||||
import zipfile
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import pytest
|
||||
from playwright.sync_api import expect, sync_playwright
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_zip64_browser_seeks_saved_output_from_any_plugin(snapshot, live_server):
|
||||
from archivebox.core.routes_util import get_snapshot_host
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
port = urlsplit(live_server.url).port
|
||||
machine = Machine.current()
|
||||
machine.config = {**machine.config, "BASE_URL": f"http://archivebox.localhost:{port}"}
|
||||
machine.save(update_fields=["config"])
|
||||
snapshot.permissions = "public"
|
||||
snapshot.save(update_fields=["permissions"])
|
||||
output = snapshot.output_dir / "staticfile" / "nested" / "contents.zip"
|
||||
output.parent.mkdir(parents=True)
|
||||
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr("folder/readme.txt", "Seekable ZIP64 preview content")
|
||||
archive.writestr("folder/page.html", '<script>document.title="unsafe"</script>')
|
||||
archive.writestr("../unsafe.txt", "never listed")
|
||||
for i in range(65536):
|
||||
archive.writestr(f"bulk/{i}.txt", "ZIP64 entry")
|
||||
host = get_snapshot_host(str(snapshot.id)).split(":")[0]
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=[f"--host-resolver-rules=MAP {host} 127.0.0.1"])
|
||||
page = browser.new_page()
|
||||
reads = []
|
||||
page.on(
|
||||
"response",
|
||||
lambda response: (
|
||||
reads.append((response.status, response.request.headers.get("range"), response.headers.get("content-length")))
|
||||
if response.request.headers.get("range")
|
||||
else None
|
||||
),
|
||||
)
|
||||
page.goto(f"http://{host}:{port}/staticfile/nested/?files=1")
|
||||
page.locator(".directory-link").filter(has_text="contents.zip").click()
|
||||
expect(page.locator("#entries")).to_contain_text("folder")
|
||||
expect(page.locator("#entries")).not_to_contain_text("unsafe.txt")
|
||||
page.get_by_role("button", name="folder", exact=False).click()
|
||||
page.get_by_role("button", name="readme.txt", exact=False).click()
|
||||
expect(page.locator("#content")).to_have_text("Seekable ZIP64 preview content")
|
||||
page.get_by_role("button", name="page.html", exact=False).click()
|
||||
expect(page.locator("#content pre")).to_contain_text("<script>")
|
||||
assert page.title() == "contents.zip"
|
||||
assert reads and all(status == 206 and byte_range for status, byte_range, _ in reads)
|
||||
assert max(int(length) for _, _, length in reads) <= 256 * 1024
|
||||
assert sum(int(length) for _, _, length in reads) < output.stat().st_size
|
||||
expect(page.locator("#error")).to_be_empty()
|
||||
browser.close()
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_directory_filter_and_row_downloads(snapshot, live_server):
|
||||
from archivebox.core.routes_util import get_snapshot_host
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
port = urlsplit(live_server.url).port
|
||||
machine = Machine.current()
|
||||
machine.config = {**machine.config, "BASE_URL": f"http://archivebox.localhost:{port}"}
|
||||
machine.save(update_fields=["config"])
|
||||
snapshot.permissions = "public"
|
||||
snapshot.save(update_fields=["permissions"])
|
||||
root = snapshot.output_dir / "dropbox" / "files"
|
||||
(root / "nested").mkdir(parents=True)
|
||||
(root / "note.txt").write_text("raw file download")
|
||||
(root / "small.html").write_text('<script>document.title="unsafe"</script><p>Inert preview</p>')
|
||||
(root / "picture.svg").write_text(
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32"><rect width="32" height="32" fill="blue"/></svg>',
|
||||
)
|
||||
(root / "large.txt").write_bytes(b"x" * (100 * 1024))
|
||||
(root / "nested" / "child.txt").write_text("folder member")
|
||||
host = get_snapshot_host(str(snapshot.id)).split(":")[0]
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=[f"--host-resolver-rules=MAP {host} 127.0.0.1"])
|
||||
page = browser.new_page(accept_downloads=True, locale="tr-TR")
|
||||
page.goto(f"http://{host}:{port}/dropbox/files/?files=1")
|
||||
expect(page.locator(".entry-download")).to_have_count(5)
|
||||
assert page.evaluate("navigator.language") == "tr-TR"
|
||||
page.get_by_role("searchbox", name="Filter files").fill("PICTURE")
|
||||
expect(page.locator(".directory-entry:not(.parent):visible .entry-name")).to_have_text(["picture.svg"])
|
||||
page.get_by_role("searchbox", name="Filter files").fill("")
|
||||
expect(page.locator('pre[data-url="note.txt?raw=1"]')).to_have_text("raw file download")
|
||||
expect(page.locator('pre[data-url="small.html?raw=1"]')).to_contain_text('<script>document.title="unsafe"</script>')
|
||||
assert page.title() != "unsafe"
|
||||
assert set(page.locator(".directory-entry").evaluate_all("rows => rows.map(row => row.getBoundingClientRect().height)")) == {50}
|
||||
assert len(set(page.locator(".entry-name").evaluate_all("names => names.map(name => name.getBoundingClientRect().left)"))) == 1
|
||||
assert page.locator("main").evaluate(
|
||||
"node => node.getBoundingClientRect().width === innerWidth && node.getBoundingClientRect().left === 0",
|
||||
)
|
||||
assert page.locator('pre[data-url="large.txt?raw=1"]').count() == 0
|
||||
page.wait_for_function('document.querySelector("img.entry-preview").naturalWidth === 32')
|
||||
expect(page.locator(".entry-type").filter(has_text="image/svg+xml")).to_have_count(1)
|
||||
names = page.locator(".directory-entry:not(.parent) .entry-name")
|
||||
page.get_by_role("button", name="Size", exact=True).click()
|
||||
expect(names).to_have_text(["nested/", "note.txt", "small.html", "picture.svg", "large.txt"])
|
||||
page.get_by_role("button", name="Size", exact=True).click()
|
||||
expect(names).to_have_text(["nested/", "large.txt", "picture.svg", "small.html", "note.txt"])
|
||||
page.get_by_role("button", name="Type", exact=True).click()
|
||||
expect(names).to_have_text(["nested/", "picture.svg", "small.html", "large.txt", "note.txt"])
|
||||
page.get_by_role("button", name="Name", exact=True).click()
|
||||
expect(names).to_have_text(["nested/", "large.txt", "note.txt", "picture.svg", "small.html"])
|
||||
page.get_by_role("button", name="Name", exact=True).click()
|
||||
expect(names).to_have_text(["nested/", "small.html", "picture.svg", "note.txt", "large.txt"])
|
||||
page.get_by_role("searchbox", name="Filter files").fill("NOTE")
|
||||
expect(page.locator(".directory-entry:not(.parent):visible")).to_have_count(1)
|
||||
expect(page.locator("#file-count")).to_have_text("Items: 1")
|
||||
with page.expect_download() as downloaded:
|
||||
page.get_by_role("link", name="Download note.txt", exact=True).click()
|
||||
assert downloaded.value.failure() is None
|
||||
from pathlib import Path
|
||||
|
||||
assert Path(downloaded.value.path()).read_text() == "raw file download"
|
||||
page.get_by_role("searchbox", name="Filter files").fill("")
|
||||
with page.expect_download() as downloaded:
|
||||
page.get_by_role("link", name="Download nested/", exact=True).click()
|
||||
assert downloaded.value.failure() is None
|
||||
with zipfile.ZipFile(downloaded.value.path()) as archive:
|
||||
assert archive.namelist() == ["nested/child.txt"]
|
||||
assert archive.read("nested/child.txt") == b"folder member"
|
||||
assert not list(root.rglob("*.zip"))
|
||||
(root / "nested" / "file10.txt").write_text("ten")
|
||||
(root / "nested" / "file2.txt").write_text("two")
|
||||
page.goto(f"http://{host}:{port}/dropbox/files/nested/?files=1")
|
||||
expect(names).to_have_text(["child.txt", "file2.txt", "file10.txt"])
|
||||
page.get_by_role("button", name="Name", exact=True).click()
|
||||
expect(names).to_have_text(["file10.txt", "file2.txt", "child.txt"])
|
||||
browser.close()
|
||||
@@ -23,12 +23,16 @@ def compact_uuid(value: uuid.UUID | str | None) -> CompactUUID | None:
|
||||
if isinstance(value, CompactUUID):
|
||||
return value
|
||||
if isinstance(value, uuid.UUID):
|
||||
return CompactUUID(hex=value.hex)
|
||||
# ORM reads already decoded the UUID. Reuse its integer instead of
|
||||
# formatting and parsing hex again for every primary/foreign key.
|
||||
return CompactUUID(int=value.int)
|
||||
return CompactUUID(str(value))
|
||||
|
||||
|
||||
class CompactUUIDField(models.UUIDField):
|
||||
def to_python(self, value):
|
||||
if value is None or isinstance(value, CompactUUID):
|
||||
return value
|
||||
return compact_uuid(super().to_python(value))
|
||||
|
||||
def from_db_value(self, value, expression, connection):
|
||||
|
||||
@@ -116,9 +116,14 @@ class ModelWithQueue(models.Model):
|
||||
def save(self, *args: Any, **kwargs: Any) -> None:
|
||||
from archivebox.machine.models import Process
|
||||
|
||||
process = Process.current()
|
||||
if self.warn_on_save_outside_runner and not self._state.adding and process.process_type != Process.TypeChoices.ORCHESTRATOR:
|
||||
root_type = getattr(process.root, "process_type", None)
|
||||
# Saving a queue row must not register a runtime process merely to
|
||||
# produce a warning. Registration probes host/network state and writes
|
||||
# logs, work that does not belong in a queue-only HTTP request.
|
||||
# Actual command/runner startup still owns Process.current().
|
||||
process = Process.current_readonly()
|
||||
process_type = process.process_type if process else Process._detect_process_type()
|
||||
if self.warn_on_save_outside_runner and not self._state.adding and process_type != Process.TypeChoices.ORCHESTRATOR:
|
||||
root_type = getattr(process.root, "process_type", None) if process else None
|
||||
if root_type != Process.TypeChoices.ORCHESTRATOR:
|
||||
caller = "<unknown>"
|
||||
frame = inspect.currentframe()
|
||||
@@ -149,7 +154,7 @@ class ModelWithQueue(models.Model):
|
||||
self.pk,
|
||||
self.status,
|
||||
self.retry_at,
|
||||
process.process_type,
|
||||
process_type,
|
||||
root_type,
|
||||
caller,
|
||||
)
|
||||
|
||||
@@ -237,9 +237,11 @@ def _live_supervisord_processes_from_db():
|
||||
process_type=Process.TypeChoices.SUPERVISORD,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pwd=str(CONSTANTS.DATA_DIR),
|
||||
).order_by("-started_at", "-created_at")
|
||||
)
|
||||
live = []
|
||||
for process in rows.iterator(chunk_size=20):
|
||||
# Release each read before OS inspection or exit-state writes; supervisor
|
||||
# cleanup must not hold a cursor while other processes enqueue work.
|
||||
for process in rows.order_by("pk").paged_iterator(chunk_size=20):
|
||||
proc = process.proc
|
||||
if proc is not None:
|
||||
live.append((process, proc))
|
||||
@@ -581,14 +583,18 @@ def _current_foreground_supervisord_watchdog_args():
|
||||
from archivebox.machine.models import Machine, Process
|
||||
|
||||
current = Process.current()
|
||||
for process in Process.objects.filter(
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.SUPERVISORD,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pwd=str(CONSTANTS.DATA_DIR),
|
||||
pid=_supervisord_proc.pid,
|
||||
parent=current,
|
||||
).iterator(chunk_size=10):
|
||||
for process in (
|
||||
Process.objects.filter(
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.SUPERVISORD,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pwd=str(CONSTANTS.DATA_DIR),
|
||||
pid=_supervisord_proc.pid,
|
||||
parent=current,
|
||||
)
|
||||
.order_by("pk")
|
||||
.paged_iterator(chunk_size=10)
|
||||
):
|
||||
if process.is_running:
|
||||
owner = psutil.Process(current.pid)
|
||||
supervisord = psutil.Process(process.pid)
|
||||
@@ -928,13 +934,17 @@ def stop_own_supervisord_process(*, record_exit: bool = True):
|
||||
try:
|
||||
from archivebox.machine.models import Machine, Process
|
||||
|
||||
for process in Process.objects.filter(
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.SUPERVISORD,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pwd=str(CONSTANTS.DATA_DIR),
|
||||
pid=stopped_pid,
|
||||
).iterator(chunk_size=10):
|
||||
for process in (
|
||||
Process.objects.filter(
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.SUPERVISORD,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pwd=str(CONSTANTS.DATA_DIR),
|
||||
pid=stopped_pid,
|
||||
)
|
||||
.order_by("pk")
|
||||
.paged_iterator(chunk_size=10)
|
||||
):
|
||||
process.mark_exited(exit_code=0)
|
||||
except _PROCESS_STATE_ERRORS as err:
|
||||
_warn_background_cleanup("Could not mark supervisord process exited", err)
|
||||
|
||||
@@ -91,7 +91,7 @@ def main():
|
||||
raise RuntimeError("Automatic deployment/capture did not complete within the acceptance window")
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch()
|
||||
browser = playwright.chromium.launch(executable_path=os.environ["CHROME_BINARY"])
|
||||
page = browser.new_page(viewport={"width": 1440, "height": 1000})
|
||||
for snapshot in report["snapshots"]:
|
||||
response = page.goto(snapshot["replay_url"], wait_until="domcontentloaded")
|
||||
|
||||
@@ -156,7 +156,6 @@ if [[ "$HAS_USEFUL_SNAPSHOT" == "0" ]]; then
|
||||
--depth=0 \
|
||||
--overwrite \
|
||||
--tag=documentation,reference \
|
||||
--plugins=title,headers,wget,screenshot,pdf,dom,readability,htmltotext,hashes,dns \
|
||||
https://docs.sweeting.me/s/cookie-dilemma https://archivebox.io
|
||||
)
|
||||
fi
|
||||
@@ -211,17 +210,10 @@ echo "[*] Creating a temporary screenshot admin"
|
||||
CREATED_TEMP_USER=1
|
||||
|
||||
# Configure the optional UI before the server snapshots its startup config.
|
||||
OPENCODE_PORT="$(uv run --no-cache --project "$REPO_DIR" python - <<'PYPORT'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PYPORT
|
||||
)"
|
||||
(
|
||||
cd "$DATA_DIR"
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox config --set "OPENCODE_PORT=$OPENCODE_PORT"
|
||||
uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'from abx_plugins.plugins.opencode.archivebox.screenshots import configure; configure()'
|
||||
)
|
||||
|
||||
echo "[*] Starting ArchiveBox on port $PORT"
|
||||
@@ -274,11 +266,11 @@ VIEWS=(
|
||||
add_supplementary_template_capture() {
|
||||
local plugin_names="$1"
|
||||
local source_url="$2"
|
||||
local prepare_plugins="$3"
|
||||
local capture_plugins="${prepare_plugins:-$plugin_names}"
|
||||
local capture_log="$CAPTURE_ROOT/${plugin_names}-supplementary-capture.log"
|
||||
local supplementary_plugins=()
|
||||
IFS=',' read -r -a supplementary_plugins <<<"$plugin_names"
|
||||
# These are real all-plugin captures. The recipe only selects which output
|
||||
# to photograph, never which extractors run or which dependencies get saved.
|
||||
echo "[*] Capturing real $plugin_names source for template coverage"
|
||||
if ! (
|
||||
cd "$DATA_DIR"
|
||||
@@ -286,7 +278,6 @@ add_supplementary_template_capture() {
|
||||
--depth=0 \
|
||||
--overwrite \
|
||||
--tag=screenshot-gallery \
|
||||
--plugins="$capture_plugins" \
|
||||
"$source_url"
|
||||
) >"$capture_log" 2>&1; then
|
||||
echo "[!] Supplementary $plugin_names capture failed" >&2
|
||||
@@ -309,20 +300,6 @@ add_supplementary_template_capture() {
|
||||
|
||||
local snapshot_id snapshot_view_url snapshot_view_path
|
||||
IFS=$'\t' read -r snapshot_id snapshot_view_url snapshot_view_path <<<"$snapshot_record"
|
||||
if [[ -n "$prepare_plugins" ]]; then
|
||||
if ! (
|
||||
cd "$DATA_DIR"
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox extract \
|
||||
--plugins="$plugin_names" \
|
||||
"$snapshot_id"
|
||||
) >>"$capture_log" 2>&1; then
|
||||
echo "[!] $plugin_names extraction failed for the prepared snapshot" >&2
|
||||
tail -100 "$capture_log" >&2
|
||||
exit 1
|
||||
fi
|
||||
stop_background_runner
|
||||
fi
|
||||
|
||||
local discovery_report="$CAPTURE_ROOT/${plugin_names}-output-discovery.json"
|
||||
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
|
||||
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
|
||||
@@ -367,6 +344,12 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
|
||||
[[ "$capture_mode" != wait-text:* ]] || wait_for_text="${capture_mode#wait-text:}"
|
||||
preview_wait_for_text=""
|
||||
[[ "$capture_mode" != wait-preview-text:* ]] || preview_wait_for_text="${capture_mode#wait-preview-text:}"
|
||||
config_selector=""
|
||||
config_click_selector=""
|
||||
if [[ "$capture_mode" == plugin-config:* ]]; then
|
||||
config_selector="[data-plugin-name='${capture_mode#plugin-config:}']"
|
||||
config_click_selector="$config_selector .plugin-config-marker"
|
||||
fi
|
||||
view_timing_report=""
|
||||
while IFS='|' read -r profile _viewport_width _viewport_height; do
|
||||
filename="$slug-$profile.png"
|
||||
@@ -490,6 +473,8 @@ PY
|
||||
SCREENSHOT_HEIGHT=1000 \
|
||||
SCREENSHOT_VARIANTS_JSON="$output_variants" \
|
||||
SCREENSHOT_COLLAPSE_FILTERS=1 \
|
||||
SCREENSHOT_CLICK_SELECTOR="$config_click_selector" \
|
||||
SCREENSHOT_SCROLL_SELECTOR="$config_selector" \
|
||||
SCREENSHOT_SNAPSHOT_HEADER="$snapshot_header" \
|
||||
SCREENSHOT_WAIT_FOR_TEXT="$wait_for_text" \
|
||||
SCREENSHOT_PREVIEW_WAIT_FOR_TEXT="$preview_wait_for_text" \
|
||||
@@ -589,7 +574,7 @@ PY
|
||||
VIEWS+=(
|
||||
"Add URLs|$ADMIN_BASE_URL/add/|/add/|archivebox/core/views.py"
|
||||
"Admin dashboard|$ADMIN_BASE_URL/admin/|/admin/|archivebox/core/admin_site.py"
|
||||
"AI agent|$ADMIN_BASE_URL/admin/agent/|/admin/agent/|archivebox/opencode/views.py|wait-text:ArchiveBox AI Agent"
|
||||
"$(uv run --no-cache --project "$REPO_DIR" python -c 'import sys; from abx_plugins.plugins.opencode.archivebox.screenshots import gallery_entry; print(gallery_entry(sys.argv[1]))' "$ADMIN_BASE_URL")"
|
||||
"Snapshots table|$ADMIN_BASE_URL/admin/core/snapshot/|/admin/core/snapshot/|archivebox/core/admin_snapshots.py"
|
||||
"Snapshots grid|$ADMIN_BASE_URL/admin/core/snapshot/grid/|/admin/core/snapshot/grid/|archivebox/templates/admin/snapshots_grid.html"
|
||||
"Snapshot admin detail|$ADMIN_BASE_URL/admin/core/snapshot/$SNAPSHOT_ID/change/|/admin/core/snapshot/$SNAPSHOT_ID/change/|archivebox/core/admin_snapshots.py"
|
||||
@@ -662,11 +647,18 @@ PY
|
||||
[[ -z "$plugin_name" ]] && continue
|
||||
VIEWS+=("Snapshot View ($plugin_name)|$LIVE_SNAPSHOT_VIEW_URL#$output_path|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|$output_capture_mode")
|
||||
done <<<"$SNAPSHOT_OUTPUT_PLUGINS"
|
||||
while IFS='|' read -r example_plugins example_url example_prepare; do
|
||||
while IFS='|' read -r example_plugins example_url; do
|
||||
[[ -z "$example_plugins" ]] && continue
|
||||
add_supplementary_template_capture "$example_plugins" "$example_url" "$example_prepare"
|
||||
add_supplementary_template_capture "$example_plugins" "$example_url"
|
||||
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json,sys; plan=json.load(open(sys.argv[1])); print("\n".join("|".join((",".join(example["plugins"]), example["url"], ",".join(example["prepare_plugins"]))) for example in plan["examples"]))' \
|
||||
'import json,sys; plan=json.load(open(sys.argv[1])); print("\n".join("|".join((",".join(example["plugins"]), example["url"])) for example in plan["examples"]))' \
|
||||
"$PLUGIN_SCREENSHOT_PLAN")
|
||||
|
||||
while IFS='|' read -r config_name config_plugin; do
|
||||
[[ -z "$config_plugin" ]] && continue
|
||||
VIEWS+=("$config_name|$ADMIN_BASE_URL/add/|/add/|archivebox/templates/plugins/plugin_config_grid.html|plugin-config:$config_plugin")
|
||||
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json,sys; print("\n".join(view["name"] + "|" + view["plugin"] for view in json.load(open(sys.argv[1]))["config_views"]))' \
|
||||
"$PLUGIN_SCREENSHOT_PLAN")
|
||||
|
||||
VIEWS+=("Snapshot View (header collapsed)|$LIVE_SNAPSHOT_VIEW_URL|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|snapshot-collapsed")
|
||||
|
||||
@@ -254,7 +254,7 @@ find /tmp "$TMP_DIR" -maxdepth 1 -type d -name "archivebox-chrome-profile.*" -mm
|
||||
|
||||
ensure_dir "/home/$ARCHIVEBOX_USER"
|
||||
# A nested read-only browser-profile mount can create this parent as root.
|
||||
# OpenCode's installer needs to create a sibling config directory as archivebox.
|
||||
# Plugin installers need to create sibling config directories as archivebox.
|
||||
ensure_dir "${XDG_CONFIG_HOME:-/home/$ARCHIVEBOX_USER/.config}"
|
||||
ensure_small_runtime_tree "$ABXBUS_CACHE_DIR"
|
||||
ensure_small_runtime_tree "$ABXBUS_CACHE_DIR/semaphores"
|
||||
|
||||
@@ -67,6 +67,7 @@ def plugin_screenshot_plan() -> dict:
|
||||
required = []
|
||||
examples = {}
|
||||
overrides = {}
|
||||
config_views = []
|
||||
enabled_plugins = []
|
||||
for plugin_dir in sorted(get_plugins_dir().iterdir()):
|
||||
config_path = plugin_dir / "config.json"
|
||||
@@ -81,6 +82,8 @@ def plugin_screenshot_plan() -> dict:
|
||||
enabled = True
|
||||
if recipe.get("view"):
|
||||
required.append(recipe["view"])
|
||||
if recipe.get("config"):
|
||||
config_views.append({"name": recipe["view"], "plugin": plugin})
|
||||
elif enabled and (
|
||||
recipe.get("url")
|
||||
or (plugin_dir / "templates/full.html").is_file()
|
||||
@@ -89,10 +92,16 @@ def plugin_screenshot_plan() -> dict:
|
||||
required.append(f"Snapshot View ({plugin})")
|
||||
if recipe.get("url"):
|
||||
overrides[plugin] = recipe
|
||||
key = (recipe["url"], tuple(recipe.get("prepare_plugins", [])))
|
||||
example = examples.setdefault(key, {"url": key[0], "prepare_plugins": list(key[1]), "plugins": []})
|
||||
key = recipe["url"]
|
||||
example = examples.setdefault(key, {"url": key, "plugins": []})
|
||||
example["plugins"].append(plugin)
|
||||
return {"required": required, "examples": list(examples.values()), "overrides": overrides, "enabled": enabled_plugins}
|
||||
return {
|
||||
"required": required,
|
||||
"examples": list(examples.values()),
|
||||
"overrides": overrides,
|
||||
"enabled": enabled_plugins,
|
||||
"config_views": config_views,
|
||||
}
|
||||
|
||||
|
||||
def build_provenance() -> dict[str, str]:
|
||||
@@ -259,7 +268,8 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
|
||||
route = parsed_url.path or "/"
|
||||
if parsed_url.fragment:
|
||||
route = f"{route}#{parsed_url.fragment}"
|
||||
source_url = f"{source_base_url}{capture['source']}"
|
||||
source = capture["source"]
|
||||
source_url = source if source.startswith("https://") else f"{source_base_url}{source}"
|
||||
ttfb_values = [variant["ttfb_ms"] for variant in variants.values() if isinstance(variant.get("ttfb_ms"), int | float)]
|
||||
ttfb_text = f" · ~{round(sum(ttfb_values) / len(ttfb_values))}ms TTFB" if ttfb_values else ""
|
||||
markdown_cells = []
|
||||
|
||||
+13
-7
@@ -229,6 +229,10 @@ async function main() {
|
||||
const height = Number(process.env.SCREENSHOT_HEIGHT || 1400);
|
||||
const fullPage = process.env.SCREENSHOT_FULL_PAGE === '1';
|
||||
const requireLiveProgress = process.env.SCREENSHOT_REQUIRE_LIVE_PROGRESS === '1';
|
||||
// Include Chrome startup in the capture deadline. Starting a new 60s budget
|
||||
// after launch lets cold/contended browser startup exceed the caller's total
|
||||
// timeout before a missing screencast can produce its intended error.
|
||||
const liveProgressDeadline = Date.now() + 60000;
|
||||
const variants = process.env.SCREENSHOT_VARIANTS_JSON
|
||||
? JSON.parse(process.env.SCREENSHOT_VARIANTS_JSON)
|
||||
: [];
|
||||
@@ -325,7 +329,6 @@ async function main() {
|
||||
await page.setCookie(cookie);
|
||||
}
|
||||
|
||||
const liveProgressDeadline = Date.now() + 60000;
|
||||
let navigationResponse = await page.goto(url, { waitUntil: 'load', timeout: 60000 });
|
||||
|
||||
if (process.env.SCREENSHOT_LOGIN_USERNAME || process.env.SCREENSHOT_LOGIN_PASSWORD) {
|
||||
@@ -371,12 +374,7 @@ async function main() {
|
||||
if (process.env.SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR, { timeout: 45000 });
|
||||
}
|
||||
if (process.env.SCREENSHOT_SCROLL_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_SCROLL_SELECTOR, { timeout: 45000 }).catch(() => {});
|
||||
await page.evaluate((selector) => {
|
||||
document.querySelector(selector)?.scrollIntoView({ block: 'start', inline: 'nearest' });
|
||||
}, process.env.SCREENSHOT_SCROLL_SELECTOR);
|
||||
}
|
||||
|
||||
|
||||
if (requireLiveProgress) {
|
||||
await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
|
||||
@@ -443,6 +441,14 @@ async function main() {
|
||||
const screenshotPaths = [];
|
||||
if (requireLiveProgress) checks.liveProgressCaptures = [];
|
||||
const capture = async (screenshotPath) => {
|
||||
// Resizing can move the target to a different row/column. Re-anchor each
|
||||
// gallery variant so it photographs the requested controls at every width.
|
||||
if (process.env.SCREENSHOT_SCROLL_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_SCROLL_SELECTOR, { timeout: 45000 }).catch(() => {});
|
||||
await page.evaluate((selector) => {
|
||||
document.querySelector(selector)?.scrollIntoView({ block: 'start', inline: 'nearest' });
|
||||
}, process.env.SCREENSHOT_SCROLL_SELECTOR);
|
||||
}
|
||||
await waitForVisibleCardPreviews(page, requireLiveProgress ? Math.max(1, liveProgressDeadline - Date.now()) : 45000);
|
||||
if (requireLiveProgress) {
|
||||
const liveProgress = await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
|
||||
|
||||
+1
-1
@@ -56,7 +56,7 @@ View: [`/admin/`](http://admin.archivebox.localhost:5797/admin/) · [View code](
|
||||
|
||||
## AI agent
|
||||
|
||||
View: [`/admin/agent/`](http://admin.archivebox.localhost:5797/admin/agent/) · [View code](https://github.com/ArchiveBox/ArchiveBox/blob/dev/abx_plugins/plugins/opencode/views.py)
|
||||
View: [`/admin/agent/`](http://admin.archivebox.localhost:5797/admin/agent/) · [View code](https://github.com/ArchiveBox/abx-plugins/blob/main/abx_plugins/plugins/opencode/archivebox/views.py)
|
||||
|
||||
<table><thead><tr>
|
||||
<th>Desktop</th><th>Tablet</th><th>Mobile</th>
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.72rc42",
|
||||
"version": "0.9.72rc105",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.72rc42"
|
||||
version = "0.9.72rc105"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@@ -78,9 +78,9 @@ dependencies = [
|
||||
### Extractor dependencies (runtime binaries resolved through abxpkg)
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.74", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abxpkg==1.13.25", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abx-plugins[opencode]==1.13.124", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.13.93", # shared ArchiveBox downloader package
|
||||
"abxpkg==1.13.33", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abx-plugins[opencode]==1.13.161", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.13.126", # shared ArchiveBox downloader package
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
"browser-cookie3>=0.20.1",
|
||||
@@ -344,7 +344,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
|
||||
|
||||
|
||||
[tool.bumpver]
|
||||
current_version = "v0.9.72rc42"
|
||||
current_version = "v0.9.72rc105"
|
||||
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
|
||||
commit_message = "bump version {old_version} -> {new_version}"
|
||||
tag_message = "{new_version}"
|
||||
|
||||
@@ -13,18 +13,18 @@ supported-markers = [
|
||||
]
|
||||
|
||||
[options]
|
||||
exclude-newer = "2026-09-29T08:28:14.349171727Z"
|
||||
exclude-newer = "2026-10-01T07:11:37.418401316Z"
|
||||
exclude-newer-span = "P5D"
|
||||
|
||||
[options.exclude-newer-package]
|
||||
abxbus = { timestamp = "2026-10-04T08:28:13.349185974Z", span = "PT1S" }
|
||||
abxbus = { timestamp = "2026-10-06T07:11:36.418414Z", span = "PT1S" }
|
||||
abx-plugins = "2100-01-01T00:00:00Z"
|
||||
abx-dl = "2100-01-01T00:00:00Z"
|
||||
abxpkg = "2100-01-01T00:00:00Z"
|
||||
|
||||
[[package]]
|
||||
name = "abx-dl"
|
||||
version = "1.13.93"
|
||||
version = "1.13.126"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -38,14 +38,14 @@ dependencies = [
|
||||
{ name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fe/70/aff87b8868bf3885b39ebc40643de788fcdb5ff87940ff598262f088be6a/abx_dl-1.13.93.tar.gz", hash = "sha256:dcc851c039e0b44783f32a79c0a7a021f7fe3831cd8672b757d2633c23bad1b8", size = 97796, upload-time = "2026-10-04T08:26:41.454Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/b7/80/f30434e2e081d33c1b46233e9b284c98cff9eb8895b5fd546204953849b7/abx_dl-1.13.126.tar.gz", hash = "sha256:bdd29b626fba7a35f39865e5769b3b6ea1145a9953f2cf3aa14379eae192c248", size = 98185, upload-time = "2026-10-06T07:10:19.966Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/61/73/350d7fb01a773c1776b70e125dad6564fcce5a12adba7614260f3aa3b68e/abx_dl-1.13.93-py3-none-any.whl", hash = "sha256:869f59ad236afb3094334423b091b102da91f2b68be63573300725b5f620b767", size = 103189, upload-time = "2026-10-04T08:26:42.951Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/3e/f666f8294a31a8acd9189d790585f453e0ccecbb0a9ed9c9163c28ba6a10/abx_dl-1.13.126-py3-none-any.whl", hash = "sha256:7eb1e7295b4d11c5225d6fcd100e51069dc1022818e0c8f50abdf7d24ba4434d", size = 103547, upload-time = "2026-10-06T07:10:21.456Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "abx-plugins"
|
||||
version = "1.13.124"
|
||||
version = "1.13.161"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -54,9 +54,9 @@ dependencies = [
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bd/41/e98428669f1f30e6bc1687ffc1dee33d05c3e226aeaea25b3847b008d3ed/abx_plugins-1.13.124.tar.gz", hash = "sha256:7f3c134bbd37c17794bf2c995074dc02f92fcb7109328946e0d26cdd5e002049", size = 463017, upload-time = "2026-10-04T08:14:21.523Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3a/9d/9e442f90b30eb34fe4be44444d399685f453d3e452961018f7f91eaca173/abx_plugins-1.13.161.tar.gz", hash = "sha256:eb80af30d259716d4c713c708197328ca7fe0ad0b7fe05c27ef3bcb8112913e3", size = 509069, upload-time = "2026-10-06T07:02:15.305Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/e1/b89f45bdc937540c5a25f286f3e2ff9540d8d64fe1833d85130d88612671/abx_plugins-1.13.124-py3-none-any.whl", hash = "sha256:cff2f134ab8babd0c8e80e601a98a921a92d00ba4506f4103b0263cd72fe9f3b", size = 675947, upload-time = "2026-10-04T08:14:22.82Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/2b/da4c917bd49e92ba059b5a93b21f49499b5d385c15db23e2bc4ced834eb9/abx_plugins-1.13.161-py3-none-any.whl", hash = "sha256:1a88ec1580fa522718cf77049fd841441bd041cea63d8e38d4d1947e8b5b7b6a", size = 739583, upload-time = "2026-10-06T07:02:12.819Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -82,7 +82,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "abxpkg"
|
||||
version = "1.13.25"
|
||||
version = "1.13.33"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -90,9 +90,9 @@ dependencies = [
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f1/28/38d96ca86ca062d651d2cb78503dc4a1b9d1cbbe35e06f6f09e1fb6cd5f7/abxpkg-1.13.25.tar.gz", hash = "sha256:a9e0defdcdb74c1d63441f03cd38027c4d42a9d9675bed9cbab166be83daa11d", size = 243371, upload-time = "2026-10-04T07:01:36.249Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/dd/6f/30d3decb924862273bf19a6674c2b8cc408424fbe05c1f5d19869226bdd4/abxpkg-1.13.33.tar.gz", hash = "sha256:ca9ecb3b37a312599573e191facf75e178394b7067e7df327acf99c32bef0c40", size = 245179, upload-time = "2026-10-05T00:16:06.849Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/40/e79cf962b8e5eee8aab2353a82ef6401fce069c94284ea85875e774ce44e/abxpkg-1.13.25-py3-none-any.whl", hash = "sha256:1a0b4d98a66251d6321416f773a15673055857c4227baf2575ca1cbf389be880", size = 254763, upload-time = "2026-10-04T07:01:34.659Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/ef/0213490bc963f473078a56f0d553deacbcef00dfa5ee80d04ca9fafb73be/abxpkg-1.13.33-py3-none-any.whl", hash = "sha256:70c089fb42a2ff06affa97fc5d2cf190b9c2a4ae9561e81e57dce3e0e2933d52", size = 256428, upload-time = "2026-10-05T00:16:05.411Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -127,7 +127,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "archivebox"
|
||||
version = "0.9.72rc42"
|
||||
version = "0.9.72rc105"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -228,10 +228,10 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "abx-dl", specifier = "==1.13.93" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.124" },
|
||||
{ name = "abx-dl", specifier = "==1.13.126" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.161" },
|
||||
{ name = "abxbus", specifier = "==2.5.74" },
|
||||
{ name = "abxpkg", specifier = "==1.13.25" },
|
||||
{ name = "abxpkg", specifier = "==1.13.33" },
|
||||
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
|
||||
{ name = "atomicwrites", specifier = "==1.4.1" },
|
||||
{ name = "base32-crockford", specifier = ">=0.3.0" },
|
||||
@@ -1732,24 +1732,24 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "psutil"
|
||||
version = "7.2.2"
|
||||
version = "7.2.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/73/cb/09e5184fb5fc0358d110fc3ca7f6b1d033800734d34cac10f4136cfac10e/psutil-7.2.1.tar.gz", hash = "sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3", size = 490253, upload-time = "2025-12-29T08:26:00.169Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/8e/f0c242053a368c2aa89584ecd1b054a18683f13d6e5a318fc9ec36582c94/psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d", size = 129624, upload-time = "2025-12-29T08:26:04.255Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/97/a58a4968f8990617decee234258a2b4fc7cd9e35668387646c1963e69f26/psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49", size = 130132, upload-time = "2025-12-29T08:26:06.228Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/6d/ed44901e830739af5f72a85fa7ec5ff1edea7f81bfbf4875e409007149bd/psutil-7.2.1-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc", size = 180612, upload-time = "2025-12-29T08:26:08.276Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/65/b628f8459bca4efbfae50d4bf3feaab803de9a160b9d5f3bd9295a33f0c2/psutil-7.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf", size = 183201, upload-time = "2025-12-29T08:26:10.622Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/c2/5fb764bd61e40e1fe756a44bd4c21827228394c17414ade348e28f83cd79/psutil-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679", size = 129716, upload-time = "2025-12-29T08:26:16.017Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c9/d2/935039c20e06f615d9ca6ca0ab756cf8408a19d298ffaa08666bc18dc805/psutil-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f", size = 130133, upload-time = "2025-12-29T08:26:18.009Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/69/19f1eb0e01d24c2b3eacbc2f78d3b5add8a89bf0bb69465bc8d563cc33de/psutil-7.2.1-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129", size = 181518, upload-time = "2025-12-29T08:26:20.241Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/6d/7e18b1b4fa13ad370787626c95887b027656ad4829c156bb6569d02f3262/psutil-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a", size = 184348, upload-time = "2025-12-29T08:26:22.215Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/cf/5180eb8c8bdf6a503c6919f1da28328bd1e6b3b1b5b9d5b01ae64f019616/psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42", size = 128137, upload-time = "2025-12-29T08:26:27.759Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/2c/78e4a789306a92ade5000da4f5de3255202c534acdadc3aac7b5458fadef/psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1", size = 128947, upload-time = "2025-12-29T08:26:29.548Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/29/f8/40e01c350ad9a2b3cb4e6adbcc8a83b17ee50dd5792102b6142385937db5/psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8", size = 154694, upload-time = "2025-12-29T08:26:32.147Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/06/e4/b751cdf839c011a9714a783f120e6a86b7494eb70044d7d81a25a5cd295f/psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6", size = 156136, upload-time = "2025-12-29T08:26:34.079Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/44/ad/bbf6595a8134ee1e94a4487af3f132cef7fce43aef4a93b49912a48c3af7/psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8", size = 148108, upload-time = "2025-12-29T08:26:36.225Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/15/dd6fd869753ce82ff64dcbc18356093471a5a5adf4f77ed1f805d473d859/psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67", size = 147402, upload-time = "2025-12-29T08:26:39.21Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in new issue
Block a user