Compare commits
91
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f6360c253f | ||
|
|
eaacfa697c | ||
|
|
3b99be7fcf | ||
|
|
07d0280307 | ||
|
|
4fe2b2b7ab | ||
|
|
e5412bfe5a | ||
|
|
916c57b42e | ||
|
|
2e6794dd9c | ||
|
|
6c67977ad6 | ||
|
|
03c6f6a311 | ||
|
|
1f710263c5 | ||
|
|
cab03ba544 | ||
|
|
2d128fdf4b | ||
|
|
dc1455ccf3 | ||
|
|
976b6e5b7d | ||
|
|
709cd07eab | ||
|
|
75dcce89dd | ||
|
|
ae82f09d1a | ||
|
|
8270556ccb | ||
|
|
3502cf4312 | ||
|
|
bd11f9541f | ||
|
|
7f326aa57d | ||
|
|
1ed926bcd8 | ||
|
|
f4c30f5b74 | ||
|
|
8362d35549 | ||
|
|
a786e79fcd | ||
|
|
29fbc9c4df | ||
|
|
d8e77d5097 | ||
|
|
4b301730cd | ||
|
|
97061fc849 | ||
|
|
1ea9315492 | ||
|
|
22c30a4f31 | ||
|
|
da0a6d2221 | ||
|
|
9315887f1d | ||
|
|
d4faa4a034 | ||
|
|
dc4d75ff2f | ||
|
|
00de9bbcbb | ||
|
|
5b25b39f7a | ||
|
|
5e069e6e1f | ||
|
|
acabe3fbd6 | ||
|
|
d0eb7c60e5 | ||
|
|
7f0298e1f4 | ||
|
|
4ee4613627 | ||
|
|
489f0a1b64 | ||
|
|
1045a0e7e8 | ||
|
|
93b70398e3 | ||
|
|
5992353dca | ||
|
|
fa3af93cf0 | ||
|
|
05a43c2c9d | ||
|
|
33797e3d6a | ||
|
|
cc6e208171 | ||
|
|
b6cfe542a5 | ||
|
|
ac6ce915b1 | ||
|
|
c7a766fb98 | ||
|
|
cdb0a3b582 | ||
|
|
9a85d8dfa2 | ||
|
|
34e4211383 | ||
|
|
43edc66452 | ||
|
|
5f44ca29a9 | ||
|
|
048fd4113a | ||
|
|
e7da938f0b | ||
|
|
f4123e80c9 | ||
|
|
b035561b85 | ||
|
|
86c30d955c | ||
|
|
f9a0cab649 | ||
|
|
813a93109b | ||
|
|
19c3239bad | ||
|
|
1b795c6c07 | ||
|
|
b9bad0abae | ||
|
|
1c0335c131 | ||
|
|
861eb0f607 | ||
|
|
5f8bd00a69 | ||
|
|
c20b875736 | ||
|
|
5e8bea8d6a | ||
|
|
b167d889b5 | ||
|
|
5a6d78566d | ||
|
|
25cb823a43 | ||
|
|
c7402195e4 | ||
|
|
29a941bd9f | ||
|
|
224da46da5 | ||
|
|
396ba92672 | ||
|
|
4ff3aa3061 | ||
|
|
62b6cf80da | ||
|
|
5fe08ce366 | ||
|
|
b2d6c5a99a | ||
|
|
bfb36aee49 | ||
|
|
04a9c98e6a | ||
|
|
815bf05e07 | ||
|
|
1e3623fee3 | ||
|
|
6c83d7e23c | ||
|
|
c1af5599a5 |
No files matched your search
@@ -10,4 +10,4 @@ Also review the inline navbar/footer in abx-dl/website/index.html and
|
||||
-->
|
||||
<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps & tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS & iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS & iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps & tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS & iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS & iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
|
||||
@@ -1,30 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discover every ArchiveBox test file and expose it as one CI job."""
|
||||
"""Discover every test once, batching measured short files."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
root = Path.cwd().resolve()
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
|
||||
|
||||
if not archivebox_tests:
|
||||
raise SystemExit("No ArchiveBox tests discovered")
|
||||
|
||||
matrix: list[dict[str, object]] = []
|
||||
# These are observed whole-job seconds, including setup. Unknown/long files,
|
||||
# optional dependencies and hosted-only requirements keep independent jobs.
|
||||
durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
|
||||
hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
|
||||
requested = os.environ.get("TEST_FILE", "")
|
||||
batches: list[list[str]] = []
|
||||
batch: list[str] = []
|
||||
batch_seconds = 0
|
||||
for path in archivebox_tests:
|
||||
test_path = path.relative_to(root).as_posix()
|
||||
duration = durations.get(test_path, 60)
|
||||
if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
|
||||
batches.append([test_path])
|
||||
continue
|
||||
if batch and (len(batch) == 8 or batch_seconds + duration > 180):
|
||||
batches.append(batch)
|
||||
batch = []
|
||||
batch_seconds = 0
|
||||
batch.append(test_path)
|
||||
batch_seconds += duration
|
||||
if batch:
|
||||
batches.append(batch)
|
||||
|
||||
matrix: list[dict[str, object]] = []
|
||||
for paths in batches:
|
||||
matrix.append(
|
||||
{
|
||||
"name": f"main/{path.stem}",
|
||||
"paths": [test_path],
|
||||
"paths_arg": test_path,
|
||||
"extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
|
||||
"count": 1,
|
||||
"name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
|
||||
"paths": paths,
|
||||
"paths_arg": " ".join(paths),
|
||||
"extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
|
||||
"count": len(paths),
|
||||
"ugnas": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Ordinary Linux tests can use either pool. Only genuine hosted-only
|
||||
# requirements need a file header; filenames never enter runner policy.
|
||||
# Keep a bounded NAS share so hosted capacity continues working in parallel.
|
||||
capacity = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
|
||||
# Start long jobs first in both pools instead of leaving several minutes of
|
||||
# work behind every short job in an alphabetically ordered hosted queue.
|
||||
matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
|
||||
eligible = [item for item in matrix if not any(root / path in hosted for path in item["paths"])]
|
||||
for item in eligible[:capacity]:
|
||||
item["ugnas"] = True
|
||||
|
||||
discovered_paths = [path for entry in matrix for path in entry["paths"]]
|
||||
if len(discovered_paths) != len(set(discovered_paths)):
|
||||
raise SystemExit("Tests were not discovered exactly once")
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
{
|
||||
"source_runs": [
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/36302511923",
|
||||
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552"
|
||||
],
|
||||
"measurement": "Whole job seconds, including setup and excluding queue time. Unbatched files use the latest complete successful run; batched files retain their individual baseline measurement. Unknown files stay independent.",
|
||||
"seconds": {
|
||||
"archivebox/tests/test_api_v1_auth_browser_session.py": 29,
|
||||
"archivebox/tests/test_api_v1_auth_check_api_token.py": 27,
|
||||
"archivebox/tests/test_api_v1_auth_get_api_token.py": 24,
|
||||
"archivebox/tests/test_api_v1_cli_add.py": 96,
|
||||
"archivebox/tests/test_api_v1_cli_remove.py": 50,
|
||||
"archivebox/tests/test_api_v1_cli_schedule.py": 41,
|
||||
"archivebox/tests/test_api_v1_cli_search.py": 26,
|
||||
"archivebox/tests/test_api_v1_cli_update.py": 100,
|
||||
"archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 64,
|
||||
"archivebox/tests/test_api_v1_core_any_id.py": 21,
|
||||
"archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 55,
|
||||
"archivebox/tests/test_api_v1_core_archiveresults.py": 32,
|
||||
"archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 203,
|
||||
"archivebox/tests/test_api_v1_core_snapshots_rss.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_snapshots.py": 28,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 23,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 28,
|
||||
"archivebox/tests/test_api_v1_core_tag_tag_id.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 27,
|
||||
"archivebox/tests/test_api_v1_core_tags_autocomplete.py": 26,
|
||||
"archivebox/tests/test_api_v1_core_tags_create.py": 21,
|
||||
"archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 25,
|
||||
"archivebox/tests/test_api_v1_core_tags_search.py": 26,
|
||||
"archivebox/tests/test_api_v1_core_tags.py": 25,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 22,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 25,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 24,
|
||||
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 86,
|
||||
"archivebox/tests/test_api_v1_crawls_crawls.py": 21,
|
||||
"archivebox/tests/test_api_v1_machine_binaries.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_binary_binary_id.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 26,
|
||||
"archivebox/tests/test_api_v1_machine_machine_current.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_machine_machine_id.py": 23,
|
||||
"archivebox/tests/test_api_v1_machine_machines.py": 27,
|
||||
"archivebox/tests/test_api_v1_personas_personas.py": 23,
|
||||
"archivebox/tests/test_api_v1_personas_sync.py": 29,
|
||||
"archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 36,
|
||||
"archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 33,
|
||||
"archivebox/tests/test_archive_result_service.py": 122,
|
||||
"archivebox/tests/test_auth_ldap.py": 44,
|
||||
"archivebox/tests/test_binary_service.py": 83,
|
||||
"archivebox/tests/test_cli_add_1.py": 179,
|
||||
"archivebox/tests/test_cli_add_2.py": 167,
|
||||
"archivebox/tests/test_cli_add_3.py": 194,
|
||||
"archivebox/tests/test_cli_add_4.py": 168,
|
||||
"archivebox/tests/test_cli_archiveresult.py": 145,
|
||||
"archivebox/tests/test_cli_binary.py": 20,
|
||||
"archivebox/tests/test_cli_config.py": 141,
|
||||
"archivebox/tests/test_cli_crawl.py": 127,
|
||||
"archivebox/tests/test_cli_extract_input.py": 205,
|
||||
"archivebox/tests/test_cli_extract.py": 113,
|
||||
"archivebox/tests/test_cli_help.py": 30,
|
||||
"archivebox/tests/test_cli_init.py": 133,
|
||||
"archivebox/tests/test_cli_install.py": 58,
|
||||
"archivebox/tests/test_cli_list.py": 177,
|
||||
"archivebox/tests/test_cli_machine.py": 50,
|
||||
"archivebox/tests/test_cli_manage.py": 48,
|
||||
"archivebox/tests/test_cli_mcp.py": 51,
|
||||
"archivebox/tests/test_cli_oneshot.py": 23,
|
||||
"archivebox/tests/test_cli_persona.py": 82,
|
||||
"archivebox/tests/test_cli_piping.py": 97,
|
||||
"archivebox/tests/test_cli_pluginmap.py": 21,
|
||||
"archivebox/tests/test_cli_process.py": 20,
|
||||
"archivebox/tests/test_cli_remove.py": 150,
|
||||
"archivebox/tests/test_cli_run_1.py": 165,
|
||||
"archivebox/tests/test_cli_run_2.py": 241,
|
||||
"archivebox/tests/test_cli_run_3.py": 162,
|
||||
"archivebox/tests/test_cli_run_4.py": 200,
|
||||
"archivebox/tests/test_cli_schedule.py": 87,
|
||||
"archivebox/tests/test_cli_search.py": 35,
|
||||
"archivebox/tests/test_cli_server.py": 251,
|
||||
"archivebox/tests/test_cli_shell.py": 32,
|
||||
"archivebox/tests/test_cli_snapshot.py": 138,
|
||||
"archivebox/tests/test_cli_status.py": 105,
|
||||
"archivebox/tests/test_cli_tag.py": 21,
|
||||
"archivebox/tests/test_cli_update_reindex_snapshots.py": 144,
|
||||
"archivebox/tests/test_cli_update.py": 148,
|
||||
"archivebox/tests/test_cli_version.py": 181,
|
||||
"archivebox/tests/test_config_DELETE_AFTER.py": 192,
|
||||
"archivebox/tests/test_config_MAX_limits.py": 34,
|
||||
"archivebox/tests/test_config_ONLY_NEW.py": 24,
|
||||
"archivebox/tests/test_config_SAVE_TITLE.py": 99,
|
||||
"archivebox/tests/test_config_URL_filters.py": 26,
|
||||
"archivebox/tests/test_core_config.py": 20,
|
||||
"archivebox/tests/test_core_forms.py": 28,
|
||||
"archivebox/tests/test_crawl_pause.py": 20,
|
||||
"archivebox/tests/test_crawl_runner.py": 45,
|
||||
"archivebox/tests/test_crawl_service.py": 34,
|
||||
"archivebox/tests/test_frozen_crawl_config.py": 47,
|
||||
"archivebox/tests/test_hook_result_lifecycle.py": 78,
|
||||
"archivebox/tests/test_hooks.py": 90,
|
||||
"archivebox/tests/test_machine_models.py": 52,
|
||||
"archivebox/tests/test_machine_service.py": 104,
|
||||
"archivebox/tests/test_migrations_04_to_09.py": 36,
|
||||
"archivebox/tests/test_migrations_07_to_09.py": 140,
|
||||
"archivebox/tests/test_migrations_08_to_09.py": 257,
|
||||
"archivebox/tests/test_migrations_fresh.py": 99,
|
||||
"archivebox/tests/test_misc_checks.py": 22,
|
||||
"archivebox/tests/test_misc_monkey_patches.py": 20,
|
||||
"archivebox/tests/test_opencode_agent_1.py": 135,
|
||||
"archivebox/tests/test_opencode_agent_2.py": 138,
|
||||
"archivebox/tests/test_opencode_agent_3.py": 140,
|
||||
"archivebox/tests/test_opencode_browser_1.py": 290,
|
||||
"archivebox/tests/test_opencode_browser_2.py": 157,
|
||||
"archivebox/tests/test_opencode_browser_3.py": 129,
|
||||
"archivebox/tests/test_orm_helpers.py": 22,
|
||||
"archivebox/tests/test_permissions.py": 21,
|
||||
"archivebox/tests/test_persona_runtime.py": 56,
|
||||
"archivebox/tests/test_postgres_backend.py": 85,
|
||||
"archivebox/tests/test_process_runtime_paths.py": 28,
|
||||
"archivebox/tests/test_process_service.py": 49,
|
||||
"archivebox/tests/test_proof_previews.py": 54,
|
||||
"archivebox/tests/test_recursive_crawl_1.py": 396,
|
||||
"archivebox/tests/test_recursive_crawl_2.py": 193,
|
||||
"archivebox/tests/test_recursive_crawl_3.py": 144,
|
||||
"archivebox/tests/test_release_notes.py": 14,
|
||||
"archivebox/tests/test_release_source_fingerprint.py": 22,
|
||||
"archivebox/tests/test_release_workflow.py": 21,
|
||||
"archivebox/tests/test_resource_admission.py": 21,
|
||||
"archivebox/tests/test_runner_watch.py": 182,
|
||||
"archivebox/tests/test_search.py": 203,
|
||||
"archivebox/tests/test_serve_static.py": 31,
|
||||
"archivebox/tests/test_server_automatic_url.py": 36,
|
||||
"archivebox/tests/test_server_security_browser.py": 241,
|
||||
"archivebox/tests/test_settings_signal_webhooks.py": 21,
|
||||
"archivebox/tests/test_setup_script.py": 21,
|
||||
"archivebox/tests/test_shutdown_util.py": 22,
|
||||
"archivebox/tests/test_snapshot_filesystem_migration.py": 29,
|
||||
"archivebox/tests/test_snapshot_output_stacks.py": 62,
|
||||
"archivebox/tests/test_snapshot_ref_lookup.py": 23,
|
||||
"archivebox/tests/test_snapshot_service.py": 41,
|
||||
"archivebox/tests/test_snapshot_url_length.py": 26,
|
||||
"archivebox/tests/test_tag_service.py": 24,
|
||||
"archivebox/tests/test_takeover_util_1.py": 122,
|
||||
"archivebox/tests/test_takeover_util_2.py": 171,
|
||||
"archivebox/tests/test_takeover_util_3.py": 165,
|
||||
"archivebox/tests/test_takeover_util_4.py": 160,
|
||||
"archivebox/tests/test_test_harness.py": 22,
|
||||
"archivebox/tests/test_ui_add_view_runtime.py": 239,
|
||||
"archivebox/tests/test_ui_add_view.py": 46,
|
||||
"archivebox/tests/test_ui_admin_apitoken.py": 23,
|
||||
"archivebox/tests/test_ui_admin_archiveresult.py": 30,
|
||||
"archivebox/tests/test_ui_admin_config_widget.py": 25,
|
||||
"archivebox/tests/test_ui_admin_crawl.py": 44,
|
||||
"archivebox/tests/test_ui_admin_links.py": 35,
|
||||
"archivebox/tests/test_ui_admin_machine.py": 29,
|
||||
"archivebox/tests/test_ui_admin_persona.py": 28,
|
||||
"archivebox/tests/test_ui_admin_snapshot.py": 288,
|
||||
"archivebox/tests/test_ui_admin_tag.py": 26,
|
||||
"archivebox/tests/test_ui_admin_user.py": 28,
|
||||
"archivebox/tests/test_ui_config_views.py": 43,
|
||||
"archivebox/tests/test_ui_live_progress.py": 303,
|
||||
"archivebox/tests/test_ui_public_snapshot.py": 138,
|
||||
"archivebox/tests/test_ui_savepagenow.py": 48,
|
||||
"archivebox/tests/test_urls.py": 179,
|
||||
"archivebox/tests/test_util.py": 20,
|
||||
"archivebox/tests/test_version_metadata.py": 24
|
||||
}
|
||||
}
|
||||
@@ -18,47 +18,68 @@ concurrency:
|
||||
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
permissions:
|
||||
contents: write
|
||||
uses: ./.github/workflows/prepare-release.yml
|
||||
secrets: inherit
|
||||
|
||||
lint:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Linters
|
||||
uses: ./.github/workflows/lint.yml
|
||||
secrets: inherit
|
||||
|
||||
install-cli-platform:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Install and CLI platform compatibility
|
||||
uses: ./.github/workflows/test.yml
|
||||
secrets: inherit
|
||||
|
||||
tests:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Discovered test matrix
|
||||
uses: ./.github/workflows/test-parallel.yml
|
||||
secrets: inherit
|
||||
|
||||
documentation:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Documentation and root tests
|
||||
uses: ./.github/workflows/docs.yml
|
||||
secrets: inherit
|
||||
|
||||
codeql:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: CodeQL
|
||||
uses: ./.github/workflows/codeql.yml
|
||||
secrets: inherit
|
||||
|
||||
python-artifacts:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Tested Python artifacts
|
||||
uses: ./.github/workflows/pip.yml
|
||||
secrets: inherit
|
||||
|
||||
docker-digests:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
name: Tested Docker images
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
push_digests: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
|
||||
push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
|
||||
secrets: inherit
|
||||
|
||||
required:
|
||||
name: All required CI lanes
|
||||
if: ${{ always() }}
|
||||
if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
|
||||
needs:
|
||||
- prepare
|
||||
- lint
|
||||
- install-cli-platform
|
||||
- tests
|
||||
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_REF: ${{ inputs.source_ref }}
|
||||
SOURCE_REF: ${{ inputs.source_ref || github.sha }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
|
||||
@@ -295,7 +295,7 @@ jobs:
|
||||
'
|
||||
|
||||
- name: Validate exact built image version and commit
|
||||
if: ${{ !inputs.push_digests }}
|
||||
if: inputs.full_tests
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 4
|
||||
max-parallel: 8
|
||||
matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
@@ -1,32 +1,33 @@
|
||||
name: Release candidate
|
||||
name: Prepare release version
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
workflow_call:
|
||||
outputs:
|
||||
run_jobs:
|
||||
description: Whether this revision owns its release version and should run expensive jobs
|
||||
value: ${{ jobs.prepare.outputs.run_jobs }}
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: release-candidate-${{ github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Prepare the version before starting expensive jobs
|
||||
permissions:
|
||||
contents: write
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
action: ${{ steps.version.outputs.action }}
|
||||
run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
submodules: true
|
||||
token: ${{ secrets.RELEASE_GH_TOKEN }}
|
||||
token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
|
||||
|
||||
- name: Keep synchronized stable branches on the released version
|
||||
id: channel
|
||||
if: github.event_name == 'push'
|
||||
shell: bash
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
@@ -45,41 +46,10 @@ jobs:
|
||||
fi
|
||||
|
||||
- id: version
|
||||
if: steps.channel.outputs.stable != 'true'
|
||||
if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
|
||||
uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
|
||||
with:
|
||||
package: archivebox
|
||||
branch: ${{ github.ref_name }}
|
||||
tag-prefix: v
|
||||
version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
|
||||
|
||||
python-artifacts:
|
||||
needs: prepare
|
||||
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
|
||||
uses: ./.github/workflows/pip.yml
|
||||
with:
|
||||
full_tests: false
|
||||
secrets: inherit
|
||||
|
||||
docker-digests:
|
||||
needs: prepare
|
||||
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
push_digests: true
|
||||
full_tests: false
|
||||
secrets: inherit
|
||||
|
||||
opencode:
|
||||
needs: prepare
|
||||
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
test_file:
|
||||
- archivebox/tests/test_opencode_agent.py
|
||||
- archivebox/tests/test_opencode_browser.py
|
||||
uses: ./.github/workflows/test-parallel.yml
|
||||
with:
|
||||
test_file: ${{ matrix.test_file }}
|
||||
secrets: inherit
|
||||
@@ -2,7 +2,7 @@ name: Release
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI, Release candidate]
|
||||
workflows: [CI]
|
||||
types: [completed]
|
||||
branches: [dev, main]
|
||||
workflow_dispatch:
|
||||
@@ -111,26 +111,14 @@ jobs:
|
||||
--jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
|
||||
[[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
|
||||
else
|
||||
CANDIDATE_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow release-candidate.yml \
|
||||
--commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
|
||||
--json databaseId,status,conclusion \
|
||||
--jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
|
||||
CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
|
||||
--commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
|
||||
--json databaseId,status,conclusion \
|
||||
--jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
|
||||
[[ -n "$CANDIDATE_RUN_ID" && -n "$CI_RUN_ID" ]] || exit 0
|
||||
if [[ "$RELEASE_BRANCH" == dev ]]; then
|
||||
DIGEST_RUN_ID="$CANDIDATE_RUN_ID"
|
||||
else
|
||||
DIGEST_RUN_ID="$CI_RUN_ID"
|
||||
fi
|
||||
fi
|
||||
if [[ "$RELEASE_BRANCH" == dev && "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
|
||||
ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"
|
||||
else
|
||||
ARTIFACT_RUN_ID="$CI_RUN_ID"
|
||||
[[ -n "$CI_RUN_ID" ]] || exit 0
|
||||
DIGEST_RUN_ID="$CI_RUN_ID"
|
||||
fi
|
||||
ARTIFACT_RUN_ID="$CI_RUN_ID"
|
||||
{
|
||||
echo 'ready=true'
|
||||
echo "artifact_run_id=$ARTIFACT_RUN_ID"
|
||||
|
||||
@@ -13,6 +13,7 @@ on:
|
||||
- pyproject.toml
|
||||
- uv.lock
|
||||
- .github/workflows/screenshots.yml
|
||||
- .github/workflows/prepare-release.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -23,7 +24,15 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
permissions:
|
||||
contents: write
|
||||
uses: ./.github/workflows/prepare-release.yml
|
||||
secrets: inherit
|
||||
|
||||
deploy:
|
||||
needs: prepare
|
||||
if: needs.prepare.outputs.run_jobs == 'true'
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
@@ -42,7 +42,9 @@ jobs:
|
||||
organization: ArchiveBox
|
||||
primary-runner: self-hosted,ugnas-linux
|
||||
fallback-runner: ubuntu-24.04
|
||||
primaries-required: 1
|
||||
# Jobs queued for this pool must not be mistaken for idle runner slots:
|
||||
# requiring MAX_JOBS idle runners prevents assigning a second wave.
|
||||
primaries-required: ${{ vars.UGNAS_CI_MIN_IDLE || 1 }}
|
||||
fallback-on-error: true
|
||||
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
@@ -87,6 +89,7 @@ jobs:
|
||||
id: discover
|
||||
env:
|
||||
TEST_FILE: ${{ inputs.test_file }}
|
||||
UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
|
||||
run: |
|
||||
set -Eeuo pipefail
|
||||
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
|
||||
@@ -102,10 +105,10 @@ jobs:
|
||||
name: ${{ matrix.test.name }}
|
||||
needs: discover-tests
|
||||
# Opt in trusted branches only; PRs always use GitHub-hosted runners.
|
||||
# UGNAS_CI_TESTS is a JSON array of matrix names, e.g. ["main/test_cli_list"].
|
||||
# Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
|
||||
# The hosted discovery job falls back when workers are offline/busy or the
|
||||
# availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
|
||||
runs-on: ${{ fromJSON(contains(fromJSON(vars.UGNAS_CI_TESTS || '[]'), matrix.test.name) && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
|
||||
runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
|
||||
env:
|
||||
CHROME_HEADLESS: "true"
|
||||
|
||||
@@ -186,13 +189,35 @@ jobs:
|
||||
for test_path in "${test_paths[@]}"; do
|
||||
test -f "$test_path"
|
||||
done
|
||||
test_status=0
|
||||
for test_path in "${test_paths[@]}"; do
|
||||
test_name="$(basename "$test_path" .py)"
|
||||
test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
|
||||
export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
|
||||
export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
|
||||
export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
|
||||
if (( ${#test_paths[@]} > 1 )); then
|
||||
export ABXPKG_LIB_DIR="$RUNNER_TEMP/archivebox-lib/$test_slug"
|
||||
fi
|
||||
mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
|
||||
uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
|
||||
--basetemp="$RUNNER_TEMP/pytest/$test_slug"
|
||||
uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" --durations=20 \
|
||||
--basetemp="$RUNNER_TEMP/pytest/$test_slug" \
|
||||
--junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
|
||||
done
|
||||
exit "$test_status"
|
||||
- name: Save test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: test-results-${{ strategy.job-index }}
|
||||
path: ${{ runner.temp }}/test-results/*.xml
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
- name: Save failed runtime diagnostics
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: runtime-logs-${{ strategy.job-index }}
|
||||
path: ${{ runner.temp }}/pytest/**/*.log
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
@@ -77,11 +77,7 @@ archivebox run
|
||||
|
||||
## Verification
|
||||
|
||||
Use targeted tests for focused work:
|
||||
|
||||
```bash
|
||||
uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
|
||||
uv run prek run --all-files
|
||||
```
|
||||
Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
|
||||
with `uv run prek run --all-files`. For focused work, select the relevant tests.
|
||||
|
||||
Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
|
||||
@@ -105,12 +105,11 @@ curl -fsSL 'https://get.archivebox.io' | bash
|
||||
</details>
|
||||
<br/>
|
||||
|
||||
|
||||
<div align="center" style="text-align: center">
|
||||
<br/><br/>
|
||||
<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
|
||||
<br/><br/>
|
||||
<small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
|
||||
<small><a href="https://demo.archivebox.io">Demo</a> | <a href="https://archivebox.io/screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
|
||||
<br/>
|
||||
<sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
|
||||
<br/><br/>
|
||||
@@ -381,7 +380,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
|
||||
> *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
|
||||
|
||||
<ul>
|
||||
<li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
|
||||
<li><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a></li>
|
||||
<li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
|
||||
<li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
|
||||
<li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
|
||||
|
||||
@@ -182,6 +182,14 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
|
||||
if not file_path.is_file() or crawl_root not in file_path.parents:
|
||||
raise HttpError(404, "Crawl file not found")
|
||||
|
||||
if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
|
||||
try:
|
||||
started_at = float(request.GET["after"])
|
||||
except ValueError:
|
||||
raise HttpError(400, "Invalid screencast start time")
|
||||
if file_path.stat().st_mtime < started_at:
|
||||
raise HttpError(404, "No frame for this capture yet")
|
||||
|
||||
response = FileResponse(file_path.open("rb"))
|
||||
response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
|
||||
response["Pragma"] = "no-cache"
|
||||
|
||||
@@ -174,8 +174,12 @@ def cli(ctx, help=False):
|
||||
archivebox_init(quick=True)
|
||||
os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
|
||||
check_data_folder()
|
||||
if subcommand != "update":
|
||||
check_migrations(auto_apply=True)
|
||||
if subcommand != "update" and not check_migrations(auto_apply=True):
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
# Host registration and config reconciliation belong to CLI
|
||||
# startup; get_config() is also used by read-only HTTP views.
|
||||
Machine.current()
|
||||
except Exception as e:
|
||||
STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
|
||||
if subcommand not in ("manage", "shell"): # not all management commands need django to be setup beforehand
|
||||
|
||||
@@ -111,6 +111,11 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
|
||||
else:
|
||||
os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
|
||||
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
# Registration requires the final schema and reconciles legacy file config.
|
||||
Machine.current()
|
||||
|
||||
from archivebox.misc.db import database_display_location, is_postgres
|
||||
|
||||
assert database_exists()
|
||||
|
||||
@@ -1113,7 +1113,7 @@ def get_config(
|
||||
if apps.ready:
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
machine = Machine.current()
|
||||
machine = Machine.current_readonly()
|
||||
except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
|
||||
machine = None
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Repair process links after both core's legacy copy and machine migrations."""
|
||||
|
||||
from importlib import import_module
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
repair_process_binary_iface_links = import_module(
|
||||
"archivebox.machine.migrations.0020_repair_process_binary_iface_links",
|
||||
).repair_process_binary_iface_links
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [
|
||||
("core", "0055_repair_legacy_retry_timestamps"),
|
||||
("machine", "0022_networkinterface_identity_without_mac"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
# The original repair can run before core.0027 creates legacy Process
|
||||
# rows. Repair again after the copy, including already upgraded DBs.
|
||||
migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -4436,6 +4436,10 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
return 0
|
||||
|
||||
def output_file_map(self) -> dict[str, dict[str, Any]]:
|
||||
# List views opt into this cache on their read-only, request-local rows.
|
||||
# Runner-owned results must still reflect metadata changed during a hook.
|
||||
if "_render_output_file_map" in self.__dict__:
|
||||
return self._render_output_file_map
|
||||
return self._normalize_output_files(self.output_files)
|
||||
|
||||
def output_file_paths(self) -> list[str]:
|
||||
@@ -4590,11 +4594,20 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
output_file_map: dict[str, dict[str, Any]] | None = None,
|
||||
) -> str | None:
|
||||
ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid"}
|
||||
candidates = [
|
||||
path
|
||||
for path in output_file_paths
|
||||
if Path(path).name not in ignored and Path(path).suffix.lower() not in (".pid", ".log", ".sh")
|
||||
]
|
||||
# Responses/wget manifests can contain thousands of paths. Parse each
|
||||
# once instead of rebuilding Path objects for every preferred filename
|
||||
# and extension group, each time a list row selects a preview.
|
||||
candidates = []
|
||||
names = {}
|
||||
extensions = {}
|
||||
for path in output_file_paths:
|
||||
parsed_path = Path(path)
|
||||
name, extension = parsed_path.name, parsed_path.suffix.lower()
|
||||
if name in ignored or extension in (".pid", ".log", ".sh"):
|
||||
continue
|
||||
candidates.append(path)
|
||||
names.setdefault(name.lower(), path)
|
||||
extensions[path] = extension
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
@@ -4617,9 +4630,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
"article.json",
|
||||
]
|
||||
for preferred_name in preferred_names:
|
||||
for candidate in candidates:
|
||||
if Path(candidate).name.lower() == preferred_name:
|
||||
return candidate
|
||||
if preferred_name in names:
|
||||
return names[preferred_name]
|
||||
|
||||
from archivebox.plugins.discovery import get_plugin_output_extension_preference
|
||||
|
||||
@@ -4630,7 +4642,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
|
||||
(".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
|
||||
)
|
||||
for ext_group in ext_groups:
|
||||
group_candidates = [candidate for candidate in candidates if Path(candidate).suffix.lower() in ext_group]
|
||||
group_candidates = [candidate for candidate in candidates if extensions[candidate] in ext_group]
|
||||
if group_candidates:
|
||||
return max(
|
||||
group_candidates,
|
||||
|
||||
@@ -74,8 +74,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
|
||||
**snapshot_filter,
|
||||
).update(retry_at=now, modified_at=now)
|
||||
|
||||
# Find rare uploads through hook_name's index before checking their parents;
|
||||
# a correlated upload probe otherwise visits every sealed snapshot.
|
||||
extension_upload_results = ArchiveResult.objects.filter(
|
||||
snapshot_id=OuterRef("pk"),
|
||||
hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
|
||||
)
|
||||
server_results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")).exclude(
|
||||
@@ -83,15 +84,15 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
|
||||
)
|
||||
extension_only_snapshots = (
|
||||
Snapshot.objects.filter(
|
||||
id__in=extension_upload_results.values("snapshot_id"),
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED],
|
||||
**snapshot_filter,
|
||||
)
|
||||
.annotate(
|
||||
has_extension_upload=Exists(extension_upload_results),
|
||||
has_server_result=Exists(server_results),
|
||||
)
|
||||
.filter(has_extension_upload=True, has_server_result=False)
|
||||
.filter(has_server_result=False)
|
||||
)
|
||||
# Older browser-extension uploads could win a race with runner startup:
|
||||
# their successful external rows made the fresh Snapshot look finished
|
||||
|
||||
@@ -1,3 +1,28 @@
|
||||
"""Shared foreground signal and child-process cleanup contracts.
|
||||
|
||||
Shutdown intent and process exit are separate: takeover must preserve the
|
||||
replacement owner's work, while a user abort must not later resume the crawl.
|
||||
Callers own that policy; this module preserves the received signal and stops
|
||||
only the process objects they explicitly supply.
|
||||
|
||||
Cooperative boundaries are essential, not just quieter exception handling.
|
||||
A real Linux CI regression interrupted Popen.poll() with SIGINT and stranded
|
||||
its wait lock. SIGKILL stopped the child (it became a zombie), but Popen.wait()
|
||||
still timed out. Raising a different exception or waiting longer cannot repair
|
||||
that interrupted critical section. Polling loops must record the signal first
|
||||
and honor it between operations; async owners may cancel their own task while
|
||||
their loop is running. Once exiting, the last-resort process cleanup must stay
|
||||
synchronous: the original loop may be closed, and creating a replacement loop
|
||||
cannot recover tasks or resources owned by it.
|
||||
|
||||
Keep immediate signal notices and repeated-signal force exit independent of
|
||||
the event bus: it may already be draining or closed during shutdown. Resumable
|
||||
pause/retry/skip belongs to the existing crawl controller and must not set
|
||||
sticky command-shutdown state. See foreground_shutdown_signals below for both
|
||||
cooperative and legacy blocking callers; do not change all callers' policy at
|
||||
once without supplying their interruption boundaries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
@@ -23,7 +48,7 @@ _active_shutdown_state: ShutdownSignalState | None = None
|
||||
|
||||
|
||||
def raise_if_shutdown_requested() -> None:
|
||||
"""Let long foreground loops honor a signal even if Python ignored it once."""
|
||||
"""Honor sticky shutdown intent at a caller-owned interruption boundary."""
|
||||
|
||||
if _active_shutdown_state and _active_shutdown_state.signal_name:
|
||||
raise KeyboardInterrupt
|
||||
@@ -48,14 +73,37 @@ def wait_popen_and_kill_children(
|
||||
timeout: float,
|
||||
kill_timeout: float = 2.0,
|
||||
) -> None:
|
||||
"""Wait for a Popen parent and then hard-kill any surviving descendants."""
|
||||
"""Reap our own Popen child and stop its explicitly captured descendants.
|
||||
|
||||
A dead child still needs its parent to reap it. Keep wait failures visible;
|
||||
bypassing Popen's lock would hide the interrupted-owner bug described above.
|
||||
Descendants must still be cleaned up when that wait fails.
|
||||
"""
|
||||
|
||||
try:
|
||||
proc.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=kill_timeout)
|
||||
kill_remaining_processes(children, timeout=kill_timeout)
|
||||
try:
|
||||
proc.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=kill_timeout)
|
||||
except subprocess.TimeoutExpired as err:
|
||||
try:
|
||||
state = psutil.Process(proc.pid).as_dict(attrs=["pid", "ppid", "status", "num_threads", "cpu_times"])
|
||||
except psutil.NoSuchProcess:
|
||||
state = {"pid": proc.pid, "status": "exited"}
|
||||
err.add_note(f"Process state after SIGKILL: {state}")
|
||||
# A locked Popen wait can time out even after the OS process exits.
|
||||
# Inspect CPython's lock only for diagnostics; never release it here.
|
||||
waitpid_lock = getattr(proc, "_waitpid_lock", None)
|
||||
err.add_note(
|
||||
f"Popen state: returncode={proc.returncode}, "
|
||||
f"waitpid_lock_locked={waitpid_lock.locked() if waitpid_lock is not None else None}, "
|
||||
f"threads={[thread.name for thread in threading.enumerate()]}",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
# A parent that cannot be reaped must not leave its descendants running.
|
||||
kill_remaining_processes(children, timeout=kill_timeout)
|
||||
|
||||
|
||||
def wait_psutil_and_kill_children(
|
||||
@@ -104,11 +152,14 @@ def foreground_shutdown_signals(
|
||||
interrupt_handlers: dict[signal.Signals, Callable[[], None]] | None = None,
|
||||
raise_on_first_signal: bool = True,
|
||||
) -> Iterator[ShutdownSignalState]:
|
||||
"""Install foreground signal handlers that print an immediate exit notice.
|
||||
"""Keep shutdown intent visible until the owning foreground command unwinds.
|
||||
|
||||
Some log-tail loops intentionally swallow KeyboardInterrupt so that callers
|
||||
can centralize cleanup in finally blocks. The handler writes the signal name
|
||||
immediately, then raises KeyboardInterrupt to break out of the blocking read.
|
||||
Signal state was added because inner loops can swallow KeyboardInterrupt
|
||||
while their caller still needs to stop its children. Cooperative callers use
|
||||
raise_on_first_signal=False and check raise_if_shutdown_requested() between
|
||||
operations, or cancel their owned async task through on_signal. Raising from
|
||||
the handler can otherwise interrupt library locks or partially sent I/O.
|
||||
Blocking callers without a cooperative boundary retain immediate exceptions.
|
||||
"""
|
||||
|
||||
global _active_shutdown_state
|
||||
|
||||
@@ -775,7 +775,9 @@ def snapshot_index_row(context, link) -> str:
|
||||
title = unescape(getattr(link, "title", "") or "")
|
||||
title_text = title or url
|
||||
tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
|
||||
tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
|
||||
tag_html = "".join(
|
||||
f'<span class="snapshot-tag" title="{escape(tag)}">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag
|
||||
)
|
||||
if tag_html:
|
||||
tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
|
||||
else:
|
||||
|
||||
+53
-20
@@ -17,7 +17,6 @@ from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
|
||||
from django.contrib.auth.mixins import UserPassesTestMixin
|
||||
from django.contrib.sessions.models import Session
|
||||
from django.core import signing
|
||||
from django.core.paginator import InvalidPage
|
||||
from django.db.models import Case, IntegerField, Q, Value, When
|
||||
from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
|
||||
from django.shortcuts import redirect, render
|
||||
@@ -48,7 +47,6 @@ from archivebox.config.configset import BaseConfigSet
|
||||
from archivebox.core.forms import AddLinkForm
|
||||
from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
|
||||
from archivebox.core.permissions import (
|
||||
PERMISSIONS_PRIVATE,
|
||||
PERMISSIONS_PUBLIC,
|
||||
PERMISSIONS_UNLISTED,
|
||||
can_view_snapshot,
|
||||
@@ -92,6 +90,7 @@ from archivebox.plugins.discovery import (
|
||||
from archivebox.plugins.forms import get_plugin_config_binary_urls
|
||||
from archivebox.plugins.views import get_config_definition_link
|
||||
from archivebox.progressmonitor.views import live_progress_view
|
||||
from archivebox.progressmonitor.collection import collection_summary
|
||||
from archivebox.search.config import (
|
||||
get_search_mode,
|
||||
get_search_mode_backend,
|
||||
@@ -1238,14 +1237,23 @@ class PublicIndexView(ListView):
|
||||
"status",
|
||||
"output_size",
|
||||
"permissions",
|
||||
# Modern replay URLs need the filesystem version and owner's name.
|
||||
# Deferring these caused four extra SELECTs per displayed snapshot.
|
||||
"fs_version",
|
||||
"crawl__created_by__username",
|
||||
)
|
||||
|
||||
def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
|
||||
target_count = page_number * page_size
|
||||
def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
|
||||
# One extra row determines Next without a full collection count. Cached
|
||||
# totals can be stale while the runner adds/removes snapshots, so they
|
||||
# must never decide whether the requested page exists.
|
||||
target_count = page_number * page_size + 1
|
||||
public_snapshots: list[Snapshot] = []
|
||||
scanned = 0
|
||||
chunk_size = max(self.public_page_scan_chunk_size, page_size)
|
||||
ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
|
||||
chunk_size = max(self.public_page_scan_chunk_size, page_size + 1)
|
||||
ordered_snapshots = (
|
||||
Snapshot.objects.select_related("crawl__created_by").order_by(*self.ordering).only(*self._base_public_snapshot_fields())
|
||||
)
|
||||
|
||||
while len(public_snapshots) < target_count:
|
||||
chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
|
||||
@@ -1261,20 +1269,31 @@ class PublicIndexView(ListView):
|
||||
if self.request.GET.get("q", default="").strip():
|
||||
return super().paginate_queryset(queryset, page_size)
|
||||
|
||||
public_count = self.get_exact_public_snapshot_count()
|
||||
paginator = self.get_paginator(range(public_count), page_size)
|
||||
page_kwarg = self.kwargs.get(self.page_kwarg)
|
||||
page_query = self.request.GET.get(self.page_kwarg)
|
||||
page_number = page_kwarg or page_query or 1
|
||||
|
||||
try:
|
||||
page = paginator.page(page_number)
|
||||
except InvalidPage as err:
|
||||
page_number = int(page_number)
|
||||
if page_number < 1:
|
||||
raise ValueError("Page must be positive")
|
||||
except (ValueError, TypeError) as err:
|
||||
raise Http404(f"Invalid page ({page_number}): {err}") from err
|
||||
|
||||
object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
|
||||
page.object_list = object_list
|
||||
return paginator, page, object_list, page.has_other_pages()
|
||||
snapshots = self._ordered_public_page_from_order_index(page_number=page_number, page_size=page_size)
|
||||
if not snapshots and page_number > 1:
|
||||
raise Http404(f"Invalid page ({page_number}): No snapshots")
|
||||
observed_count = (page_number - 1) * page_size + len(snapshots)
|
||||
self.public_snapshot_count = observed_count
|
||||
if len(snapshots) > page_size:
|
||||
summary = collection_summary(self.request.user, public=True)
|
||||
cached_count = summary["snapshots"] if summary is not None else None
|
||||
self.public_snapshot_count = cached_count if cached_count is not None and cached_count >= observed_count else None
|
||||
|
||||
paginator = self.get_paginator(range(observed_count), page_size)
|
||||
page = paginator.page(page_number)
|
||||
page.object_list = snapshots[:page_size]
|
||||
return paginator, page, page.object_list, page.has_other_pages()
|
||||
|
||||
def get_context_data(self, **kwargs):
|
||||
runtime_config = self.__dict__.get("runtime_config")
|
||||
@@ -1303,10 +1322,16 @@ class PublicIndexView(ListView):
|
||||
and search_mode_backend
|
||||
and context["paginator"].count == 0,
|
||||
)
|
||||
context["public_snapshot_count"] = context["paginator"].count if query else self.public_snapshot_count
|
||||
count = context["public_snapshot_count"]
|
||||
context["public_page_count"] = (
|
||||
max(1, (count + context["paginator"].per_page - 1) // context["paginator"].per_page) if count is not None else None
|
||||
)
|
||||
snapshots = list(context.get("object_list") or ())
|
||||
from archivebox.plugins.output_groups import plugin_output_sizes
|
||||
|
||||
all_results_by_snapshot = {str(snapshot.id): [] for snapshot in snapshots}
|
||||
snapshots_by_id = {str(snapshot.id): snapshot for snapshot in snapshots}
|
||||
icons_by_snapshot: dict[str, dict[str, ArchiveResult]] = {str(snapshot.id): {} for snapshot in snapshots}
|
||||
tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
|
||||
list_icon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
|
||||
@@ -1343,8 +1368,17 @@ class PublicIndexView(ListView):
|
||||
"output_size",
|
||||
)
|
||||
)
|
||||
for result in results.iterator(chunk_size=1000):
|
||||
# This is already bounded by the displayed snapshot page. Finish
|
||||
# the SELECT before parsing output manifests: a streaming cursor
|
||||
# otherwise keeps SQLite's read lock throughout rendering.
|
||||
for result in list(results):
|
||||
snapshot_key = str(result.snapshot_id)
|
||||
# Thumbnail URLs traverse result.snapshot too; share the page's
|
||||
# already-loaded parent rather than fetching it once per result.
|
||||
result.snapshot = snapshots_by_id[snapshot_key]
|
||||
# These read-only page objects are shared by icons, sizes, and
|
||||
# thumbnails. Normalize each manifest once for this render.
|
||||
result._render_output_file_map = result.output_file_map()
|
||||
all_results_by_snapshot[snapshot_key].append(result)
|
||||
progress = progress_by_snapshot[snapshot_key]
|
||||
progress["total"] += 1
|
||||
@@ -1376,13 +1410,12 @@ class PublicIndexView(ListView):
|
||||
context["object_list"] = snapshots
|
||||
return context
|
||||
|
||||
def get_exact_public_snapshot_count(self) -> int:
|
||||
hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
|
||||
hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
|
||||
return Snapshot.objects.count() - hidden_count
|
||||
|
||||
def get_queryset(self, **kwargs):
|
||||
qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
|
||||
qs = (
|
||||
public_snapshots_queryset(super().get_queryset(**kwargs))
|
||||
.select_related("crawl__created_by")
|
||||
.only(*self._base_public_snapshot_fields())
|
||||
)
|
||||
query = self.request.GET.get("q", default="").strip()
|
||||
|
||||
if not query:
|
||||
|
||||
@@ -717,7 +717,7 @@ class InlineTagEditorWidget(TagEditorWidget):
|
||||
)
|
||||
pills_html += f'''
|
||||
<span class="tag-pill" data-tag="{self._escape(td["name"])}" data-tag-id="{td["id"]}" style="{self._tag_style(td["name"])}">
|
||||
<a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link">{self._escape(td["name"])}</a>
|
||||
<a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link" title="{self._escape(td["name"])}">{self._escape(td["name"])}</a>
|
||||
{remove_button}
|
||||
</span>
|
||||
'''
|
||||
|
||||
@@ -221,35 +221,21 @@ class Machine(ModelWithHealthStats):
|
||||
app_label = "machine"
|
||||
|
||||
@classmethod
|
||||
def current(cls, refresh: bool = False) -> Machine:
|
||||
global _CURRENT_MACHINE
|
||||
if refresh:
|
||||
_CURRENT_MACHINE = None
|
||||
if _CURRENT_MACHINE:
|
||||
if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
|
||||
# Reconciliation can save() and clear the shared cache; keep
|
||||
# this call's instance so we never return None after that save.
|
||||
machine = _CURRENT_MACHINE
|
||||
# One-time-per-process reconciliation between ArchiveBox.conf
|
||||
# and Machine.config. Fast-path: bool check + early-return when
|
||||
# the sync has already run, so the cached-machine return path
|
||||
# stays sub-microsecond.
|
||||
try:
|
||||
from archivebox.config.collection import sync_machine_and_file
|
||||
def current_readonly(cls) -> Machine | None:
|
||||
"""Read the current machine without registration, refresh, or config writes."""
|
||||
if _CURRENT_MACHINE is not None:
|
||||
return _CURRENT_MACHINE
|
||||
return cls.objects.filter(guid=get_host_guid()).first()
|
||||
|
||||
sync_machine_and_file(machine)
|
||||
except Exception:
|
||||
pass
|
||||
return machine
|
||||
else:
|
||||
_CURRENT_MACHINE = None
|
||||
@classmethod
|
||||
def current(cls) -> Machine:
|
||||
global _CURRENT_MACHINE
|
||||
if _CURRENT_MACHINE is not None:
|
||||
return _CURRENT_MACHINE
|
||||
|
||||
host_guid = get_host_guid()
|
||||
# save() intentionally invalidates _CURRENT_MACHINE so config edits
|
||||
# made through other instances become visible in this process. Host
|
||||
# refresh (after seven days), sanitation, and reconciliation can all
|
||||
# save here, so the working instance must not live only in that cache.
|
||||
# Otherwise a routine refresh leaves startup dereferencing None.
|
||||
# Host detection, sanitation, and config reconciliation run only on
|
||||
# initial registration, never on cached reads.
|
||||
try:
|
||||
machine = cls.objects.get(guid=host_guid)
|
||||
except cls.DoesNotExist:
|
||||
@@ -296,17 +282,14 @@ class Machine(ModelWithHealthStats):
|
||||
],
|
||||
)
|
||||
machine = cls._sanitize_config(machine)
|
||||
# Same one-time sync as the cached-return path. Triggers here on the
|
||||
# very first ``Machine.current()`` call in a process before any
|
||||
# cached return can occur.
|
||||
# Reconcile once, on the first Machine.current() call in this process.
|
||||
try:
|
||||
from archivebox.config.collection import sync_machine_and_file
|
||||
|
||||
sync_machine_and_file(machine)
|
||||
except Exception:
|
||||
pass
|
||||
# Publish only after all save-capable steps; publishing earlier would
|
||||
# let those steps invalidate the cache we are in the middle of filling.
|
||||
# Publish only after initialization is complete.
|
||||
_CURRENT_MACHINE = machine
|
||||
return machine
|
||||
|
||||
@@ -377,18 +360,15 @@ class Machine(ModelWithHealthStats):
|
||||
if update_fields is not None:
|
||||
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
|
||||
|
||||
# Drop the ``Machine.current()`` module-level cache on every save so
|
||||
# config edits (admin form, from_json, etc.) become live in the same
|
||||
# process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
|
||||
# The cache reads ``_CURRENT_MACHINE.modified_at`` and that value
|
||||
# never moves forward on the cached object even when the row is
|
||||
# updated in the DB, so without this we'd keep serving stale
|
||||
# ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
|
||||
# Apply explicit edits to the cached instance without making the next
|
||||
# read repeat host detection, sanitation, or config reconciliation.
|
||||
update_fields = kwargs.get("update_fields")
|
||||
super().save(*args, **kwargs)
|
||||
global _CURRENT_MACHINE
|
||||
if _CURRENT_MACHINE is not None and _CURRENT_MACHINE.pk == self.pk:
|
||||
_CURRENT_MACHINE = None
|
||||
for field in self._meta.concrete_fields:
|
||||
if field.attname in self.__dict__ and (update_fields is None or field.name in update_fields):
|
||||
setattr(_CURRENT_MACHINE, field.attname, self.__dict__[field.attname])
|
||||
|
||||
# Mirror Machine.config into ArchiveBox.conf so the two stores stay
|
||||
# 1:1. Skipped when ``update_fields`` is set and doesn't touch
|
||||
@@ -443,7 +423,7 @@ class NetworkInterface(ModelWithHealthStats):
|
||||
@classmethod
|
||||
def current(cls, refresh: bool = False) -> NetworkInterface:
|
||||
global _CURRENT_INTERFACE
|
||||
machine = Machine.current(refresh=refresh)
|
||||
machine = Machine.current()
|
||||
if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
|
||||
if not refresh:
|
||||
# Callers that pass refresh=False are asking for attribution to
|
||||
@@ -1778,7 +1758,8 @@ class Process(ModelWithDeleteAfter, models.Model):
|
||||
return None
|
||||
|
||||
# Can't validate processes on other machines
|
||||
if self.machine_id != Machine.current().id:
|
||||
machine = Machine.current_readonly()
|
||||
if machine is None or self.machine_id != machine.id:
|
||||
return None
|
||||
|
||||
try:
|
||||
|
||||
@@ -526,6 +526,8 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
|
||||
("core", "0053_remove_snapshottag_snapshot_old"),
|
||||
("core", "0054_alter_snapshot_timestamp"),
|
||||
("core", "0055_alter_tag_slug"),
|
||||
# Dev-only progress indexes removed in favor of existing-index queries.
|
||||
("core", "0056_progress_covering_indexes"),
|
||||
("core", "0056_remove_tag_uuid"),
|
||||
("core", "0057_rename_id_tag_old_id"),
|
||||
("core", "0058_alter_tag_old_id"),
|
||||
|
||||
@@ -12,9 +12,62 @@ from django.db.models import Count, Sum
|
||||
_refresh_lock = Lock()
|
||||
|
||||
|
||||
def collection_summary(user):
|
||||
def crawl_summary(crawl_id, *, snapshot_id=None):
|
||||
"""Sample sealed snapshot metadata without adding table scans to polls."""
|
||||
key = f"progress-crawl:{crawl_id}:{snapshot_id or 'all'}"
|
||||
summary = cache.get(key)
|
||||
if (summary is None or time() - summary["sampled_at"] >= 30) and _refresh_lock.acquire(blocking=False):
|
||||
if not cache.add(f"{key}:refresh", True, timeout=30):
|
||||
_refresh_lock.release()
|
||||
return summary
|
||||
|
||||
def refresh():
|
||||
try:
|
||||
close_old_connections()
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
snapshots = Snapshot.objects.filter(crawl_id=crawl_id, status=Snapshot.StatusChoices.SEALED)
|
||||
if snapshot_id is not None:
|
||||
snapshots = snapshots.filter(pk=snapshot_id)
|
||||
snapshots = snapshots.order_by("modified_at").values_list("modified_at", "downloaded_at", "output_size")
|
||||
totals = {"snapshots": 0, "cancelled": 0, "bytes": 0}
|
||||
modified_at, offset = None, 0
|
||||
while True:
|
||||
batch_scope = snapshots.filter(modified_at__gte=modified_at) if modified_at is not None else snapshots
|
||||
# Use (crawl, status, modified_at), releasing the SQLite read
|
||||
# cursor after each small batch. Never hold a cursor over the
|
||||
# whole crawl, even in this background worker.
|
||||
batch = list(batch_scope[offset : offset + 100])
|
||||
for _, downloaded_at, size in batch:
|
||||
totals["snapshots"] += 1
|
||||
totals["cancelled"] += downloaded_at is None
|
||||
totals["bytes"] += size or 0
|
||||
if len(batch) < 100:
|
||||
break
|
||||
last_modified_at = batch[-1][0]
|
||||
# Only offset within ties; otherwise advance the indexed
|
||||
# timestamp range instead of rescanning previous pages.
|
||||
offset = (offset if last_modified_at == modified_at else 0) + sum(row[0] == last_modified_at for row in batch)
|
||||
modified_at = last_modified_at
|
||||
totals["sampled_at"] = time()
|
||||
cache.set(key, totals, timeout=3600)
|
||||
except Exception:
|
||||
logging.getLogger(__name__).exception("Could not refresh crawl totals")
|
||||
finally:
|
||||
connections.close_all()
|
||||
_refresh_lock.release()
|
||||
|
||||
try:
|
||||
Thread(target=refresh, name="crawl-summary", daemon=True).start()
|
||||
except RuntimeError:
|
||||
_refresh_lock.release()
|
||||
logging.getLogger(__name__).exception("Could not start crawl totals refresh")
|
||||
return summary
|
||||
|
||||
|
||||
def collection_summary(user, *, public=False):
|
||||
user_id = None if user.is_superuser else user.pk
|
||||
key = f"progress-collection:{'all' if user_id is None else user_id}"
|
||||
key = "progress-collection:public" if public else f"progress-collection:{'all' if user_id is None else user_id}"
|
||||
summary = cache.get(key)
|
||||
# Even an indexed SUM scans the index. Never make an HTTP request wait for
|
||||
# that scan on a large collection. Serve stale totals while one worker refreshes;
|
||||
@@ -32,12 +85,23 @@ def collection_summary(user):
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
snapshots = Snapshot.objects.all()
|
||||
if user_id is not None:
|
||||
snapshots = snapshots.filter(crawl__created_by_id=user_id)
|
||||
# COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
|
||||
# Snapshot.output_size is maintained when ArchiveResult sizes change.
|
||||
totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
|
||||
totals["bytes"] = totals["bytes"] or 0
|
||||
if public:
|
||||
from archivebox.core.permissions import PERMISSIONS_PUBLIC
|
||||
|
||||
# SQLite can choose a table scan for equality after ANALYZE
|
||||
# on an all-public archive. Equal range bounds keep the count
|
||||
# on the existing permissions index. It is still O(n), so
|
||||
# public pages share this background cache, never await it.
|
||||
totals = snapshots.filter(permissions__gte=PERMISSIONS_PUBLIC, permissions__lte=PERMISSIONS_PUBLIC).aggregate(
|
||||
snapshots=Count("permissions"),
|
||||
)
|
||||
else:
|
||||
if user_id is not None:
|
||||
snapshots = snapshots.filter(crawl__created_by_id=user_id)
|
||||
# COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
|
||||
# Snapshot.output_size is maintained when ArchiveResult sizes change.
|
||||
totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
|
||||
totals["bytes"] = totals.get("bytes") or 0
|
||||
totals["sampled_at"] = time()
|
||||
cache.set(key, totals, timeout=3600)
|
||||
except Exception:
|
||||
|
||||
@@ -7,12 +7,11 @@ from typing import Literal
|
||||
from abx_dl.events import PROCESS_EXIT_SKIPPED
|
||||
from django.conf import settings
|
||||
from django.db import DatabaseError
|
||||
from django.db.models import CharField, Count, Q, Sum
|
||||
from django.db.models import CharField, Count, Q
|
||||
from django.db.models.functions import Cast
|
||||
from django.http import HttpResponse, JsonResponse
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.config.common import get_config
|
||||
from archivebox.core.permissions import can_view_snapshot, is_admin_user
|
||||
from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_url, get_api_base_url
|
||||
from archivebox.misc.logging_util import printable_filesize
|
||||
@@ -182,53 +181,37 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
|
||||
running_screencasts = {}
|
||||
|
||||
def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
|
||||
frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
|
||||
try:
|
||||
frame_stat = frame_path.stat()
|
||||
except OSError:
|
||||
return ""
|
||||
if frame_stat.st_size <= 0:
|
||||
return ""
|
||||
def screencast_frame_url(crawl_id: str) -> str:
|
||||
started_at = running_screencasts.get(crawl_id)
|
||||
if started_at is None or frame_stat.st_mtime < started_at.timestamp():
|
||||
if started_at is None:
|
||||
return ""
|
||||
rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
|
||||
# File existence/freshness belongs to the image request, never to
|
||||
# the one-second JSON poll (archive storage may be remote).
|
||||
rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={int(now.timestamp())}&after={started_at.timestamp()}"
|
||||
return f"{api_base}{rel}" if api_base else rel
|
||||
|
||||
machine_id = Machine.current().id
|
||||
machine = Machine.current_readonly()
|
||||
machine_id = machine.id if machine is not None else None
|
||||
orchestrator_proc = (
|
||||
Process.objects.filter(
|
||||
machine_id=machine_id,
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
.only("id", "pid", "started_at", "machine_id", "process_type", "status")
|
||||
.only("id", "pid", "started_at", "machine_id", "process_type", "status", "env", "cmd")
|
||||
.order_by("-started_at")
|
||||
.first()
|
||||
if machine_id is not None
|
||||
else None
|
||||
)
|
||||
runner_worker = None
|
||||
orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
|
||||
if not orchestrator_proc_running:
|
||||
try:
|
||||
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
|
||||
|
||||
supervisor = get_existing_supervisord_process(quiet=True)
|
||||
runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
|
||||
except (OSError, RuntimeError, TimeoutError):
|
||||
runner_worker = None
|
||||
|
||||
runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
|
||||
runner_worker_pid = runner_worker.get("pid") if runner_worker else None
|
||||
orchestrator_running = orchestrator_proc_running or runner_worker_running
|
||||
orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
|
||||
orchestrator_running = orchestrator_proc_running
|
||||
orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else None
|
||||
|
||||
# Get model counts by status
|
||||
crawl_status_counts = Crawl.status_counts(
|
||||
crawl_scope,
|
||||
(Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
|
||||
(Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED),
|
||||
)
|
||||
crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
|
||||
crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
|
||||
@@ -242,7 +225,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
|
||||
snapshot_status_counts = Snapshot.status_counts(
|
||||
snapshot_scope,
|
||||
Snapshot.OPEN_STATES,
|
||||
Snapshot.RUNNABLE_STATES,
|
||||
)
|
||||
snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
|
||||
snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
|
||||
@@ -252,15 +235,24 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
ArchiveResult.StatusChoices.QUEUED,
|
||||
ArchiveResult.StatusChoices.STARTED,
|
||||
)
|
||||
archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
|
||||
# Start from open results using the existing status index. Filtering by
|
||||
# plugin first scans historical runs, including all completed results.
|
||||
result_counts = list(
|
||||
archiveresult_scope.filter(status__in=result_statuses).order_by().values("status", "plugin").annotate(count=Count("*")),
|
||||
)
|
||||
archiveresult_status_counts = {
|
||||
status: sum(row["count"] for row in result_counts if row["status"] == status) for status in result_statuses
|
||||
}
|
||||
download_scope = archiveresult_scope.filter(
|
||||
plugin__in=download_plugin_names,
|
||||
snapshot__status__in=Snapshot.RUNNABLE_STATES,
|
||||
snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
|
||||
)
|
||||
indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
|
||||
download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
|
||||
indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
|
||||
indexing_status_counts = {
|
||||
status: sum(row["count"] for row in result_counts if row["status"] == status and row["plugin"] in indexing_plugin_names)
|
||||
for status in result_statuses
|
||||
}
|
||||
archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
|
||||
archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
|
||||
|
||||
@@ -296,21 +288,27 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
.values(*active_crawl_fields)
|
||||
.order_by("-modified_at")[:max_active_crawls],
|
||||
)
|
||||
paused_crawls = list(
|
||||
crawl_scope.filter(
|
||||
Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
|
||||
| Q(
|
||||
status=Crawl.StatusChoices.PAUSED,
|
||||
snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
|
||||
snapshot_set__retry_at__lte=now,
|
||||
)
|
||||
| Q(
|
||||
status=Crawl.StatusChoices.PAUSED,
|
||||
snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
|
||||
),
|
||||
# Start with runnable children, not every historical snapshot belonging
|
||||
# to a paused crawl. These uncorrelated subqueries run once per poll.
|
||||
due_paused_snapshots = (
|
||||
Snapshot.objects.filter(
|
||||
status__in=Snapshot.RUNNABLE_STATES,
|
||||
retry_at__lte=now,
|
||||
)
|
||||
.order_by()
|
||||
.values("crawl_id")
|
||||
)
|
||||
queued_paused_results = (
|
||||
ArchiveResult.objects.filter(
|
||||
status=ArchiveResult.StatusChoices.QUEUED,
|
||||
)
|
||||
.order_by()
|
||||
.values("snapshot__crawl_id")
|
||||
)
|
||||
paused_crawls = list(
|
||||
crawl_scope.filter(status=Crawl.StatusChoices.PAUSED)
|
||||
.filter(Q(created_at__gte=paused_crawl_cutoff) | Q(id__in=due_paused_snapshots) | Q(id__in=queued_paused_results))
|
||||
.values(*active_crawl_fields)
|
||||
.distinct()
|
||||
.order_by("-modified_at")[:max_active_crawls],
|
||||
)
|
||||
queued_crawls = list(
|
||||
@@ -324,52 +322,29 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
crawl["persona_id"] = str(crawl["persona_id"])
|
||||
persona_details_by_id: dict[str, dict[str, str]] = {}
|
||||
persona_details_by_name: dict[str, dict[str, str]] = {}
|
||||
persona_objects_by_id = {}
|
||||
persona_objects_by_name = {}
|
||||
persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
|
||||
persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
|
||||
if persona_ids or persona_names:
|
||||
from archivebox.personas.models import Persona
|
||||
|
||||
for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
|
||||
for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name"):
|
||||
persona_details = {
|
||||
"name": persona.name,
|
||||
"admin_url": f"/admin/personas/persona/{persona.pk}/change/",
|
||||
}
|
||||
persona_details_by_id[str(persona.id)] = persona_details
|
||||
persona_details_by_name[persona.name] = persona_details
|
||||
persona_objects_by_id[str(persona.id)] = persona
|
||||
persona_objects_by_name[persona.name] = persona
|
||||
active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
|
||||
active_crawl_objects = {}
|
||||
if active_crawl_ids:
|
||||
for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
|
||||
crawl_obj._runtime_config = request_config
|
||||
active_crawl_objects[str(crawl_obj.id)] = crawl_obj
|
||||
snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
|
||||
cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
|
||||
crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
|
||||
queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
|
||||
active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
|
||||
if active_crawl_ids:
|
||||
for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
|
||||
snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
|
||||
|
||||
for row in (
|
||||
active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
|
||||
.values("crawl_id")
|
||||
.annotate(count=Count("id"))
|
||||
):
|
||||
cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
|
||||
|
||||
for row in (
|
||||
active_snapshot_scope.filter(
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
)
|
||||
.values("crawl_id")
|
||||
.annotate(size=Sum("output_size"))
|
||||
):
|
||||
crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
|
||||
# The existing (crawl, status, modified_at) index covers these counts.
|
||||
# Byte totals and cancellation details require table reads, so sample
|
||||
# those separately and share cached values between viewer tabs.
|
||||
for row in active_snapshot_scope.order_by().values("crawl_id", "status").annotate(count=Count("*")):
|
||||
crawl_id = str(row["crawl_id"])
|
||||
snapshot_counts_by_crawl[crawl_id][row["status"]] = row["count"]
|
||||
|
||||
crawl_process_pids: dict[str, int] = {}
|
||||
snapshot_process_pids: dict[str, int] = {}
|
||||
@@ -588,7 +563,14 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
|
||||
started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
|
||||
pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
|
||||
cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
|
||||
from archivebox.progressmonitor.collection import crawl_summary
|
||||
|
||||
totals = (
|
||||
crawl_summary(crawl_id, snapshot_id=scoped_snapshot.pk if scoped_snapshot is not None else None)
|
||||
if completed_snapshots
|
||||
else {"snapshots": 0, "cancelled": 0, "bytes": 0}
|
||||
)
|
||||
cancelled_snapshots = totals["cancelled"] if totals is not None else None
|
||||
|
||||
# Count URLs in the crawl (for when snapshots haven't been created yet)
|
||||
urls_count = 0
|
||||
@@ -607,7 +589,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
|
||||
crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
|
||||
crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
|
||||
crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
|
||||
crawl_screencast_url = screencast_frame_url(crawl_id)
|
||||
crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
|
||||
|
||||
# Get active snapshots for this crawl (already prefetched)
|
||||
@@ -661,7 +643,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
snapshot_preview_url = snapshot_favicon_url
|
||||
|
||||
if snapshot["status"] == Snapshot.StatusChoices.STARTED:
|
||||
snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
|
||||
snapshot_screencast_url = screencast_frame_url(crawl_id)
|
||||
snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
|
||||
|
||||
def plugin_sort_key(ar):
|
||||
@@ -809,18 +791,18 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
|
||||
persona_name = persona_details["name"] if persona_details else "Default"
|
||||
persona_details = persona_details or persona_details_by_name.get(persona_name)
|
||||
crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
|
||||
avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
|
||||
crawl_obj = active_crawl_objects[crawl_id]
|
||||
effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
|
||||
max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
|
||||
crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
|
||||
crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
|
||||
snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
|
||||
crawl_output_size = totals["bytes"] if totals is not None else None
|
||||
avg_snapshot_size = int(totals["bytes"] / max(totals["snapshots"], 1)) if totals is not None else None
|
||||
# These crawl-scope limits are frozen in Crawl.config. Rendering
|
||||
# them must not resolve personas, probe binaries, or reread config.
|
||||
max_urls = int(crawl["config"].get("CRAWL_MAX_URLS", request_config.CRAWL_MAX_URLS) or 0)
|
||||
crawl_max_size = int(crawl["config"].get("CRAWL_MAX_SIZE", request_config.CRAWL_MAX_SIZE) or 0)
|
||||
crawl_timeout = int(crawl["config"].get("CRAWL_TIMEOUT", request_config.CRAWL_TIMEOUT) or 0)
|
||||
snapshot_max_size = int(crawl["config"].get("SNAPSHOT_MAX_SIZE", request_config.SNAPSHOT_MAX_SIZE) or 0)
|
||||
|
||||
# Check if retry_at is in the future (would prevent worker from claiming)
|
||||
retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
|
||||
is_paused = crawl_obj.is_paused
|
||||
is_paused = crawl["status"] == Crawl.StatusChoices.PAUSED
|
||||
seconds_until_retry = (
|
||||
0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
|
||||
)
|
||||
@@ -860,8 +842,8 @@ def live_progress_view(request, *, authorized_snapshot=None):
|
||||
"max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
|
||||
"crawl_output_size": crawl_output_size,
|
||||
"avg_snapshot_size": avg_snapshot_size,
|
||||
"crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
|
||||
"avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
|
||||
"crawl_output_size_display": printable_filesize(crawl_output_size or 0) if totals is not None else "…",
|
||||
"avg_snapshot_size_display": printable_filesize(avg_snapshot_size or 0) if totals is not None else "…",
|
||||
"tags": crawl_tags,
|
||||
"urls_count": urls_count,
|
||||
"total_snapshots": total_snapshots,
|
||||
|
||||
@@ -130,8 +130,8 @@ class ResourceAdmission:
|
||||
slots = min(slots, unpromised // observed_cost)
|
||||
return max(1, slots) if active == 0 and configured else slots
|
||||
|
||||
def memory_headroom(self) -> tuple[int, int, int] | None:
|
||||
"""Return workload RAM+swap usage, effective headroom, and host headroom."""
|
||||
def memory_headroom(self) -> tuple[int, int] | None:
|
||||
"""Return workload RAM+swap usage and effective available headroom."""
|
||||
try:
|
||||
meminfo = {
|
||||
name: int(value.split()[0]) * 1024
|
||||
@@ -173,7 +173,7 @@ class ResourceAdmission:
|
||||
used_bytes += child.memory_info().rss
|
||||
except (psutil.Error, OSError):
|
||||
continue
|
||||
return used_bytes, available, host_available
|
||||
return used_bytes, available
|
||||
|
||||
def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
|
||||
previous = self._memory_stalls.get(source)
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import shutil
|
||||
@@ -233,7 +234,7 @@ class CrawlRunner:
|
||||
self.snapshot_semaphore = asyncio.Semaphore(1)
|
||||
self.max_concurrent_snapshots = 1
|
||||
self._warmup_snapshot_id: str | None = None
|
||||
self._memory_baseline: tuple[int, int] | None = None
|
||||
self._memory_baseline: int | None = None
|
||||
self._memory_peak_cost = 0
|
||||
self._observed_snapshot_cost: int | None = None
|
||||
self._resource_deferred = False
|
||||
@@ -506,7 +507,7 @@ class CrawlRunner:
|
||||
return
|
||||
if self._observed_snapshot_cost is None and self._warmup_snapshot_id is None:
|
||||
observation = resource_admission.memory_headroom()
|
||||
self._memory_baseline = (observation[0], observation[2]) if observation is not None else None
|
||||
self._memory_baseline = observation[0] if observation is not None else None
|
||||
self._memory_peak_cost = 0
|
||||
self._warmup_snapshot_id = snapshot_id
|
||||
if isinstance(current_event, CrawlStartEvent):
|
||||
@@ -596,15 +597,13 @@ class CrawlRunner:
|
||||
return
|
||||
observation = resource_admission.memory_headroom()
|
||||
if observation is not None:
|
||||
used, _available, host_available = observation
|
||||
baseline_used, baseline_host_available = self._memory_baseline
|
||||
# Host headroom sees helpers outside ArchiveBox's cgroup. Keep a
|
||||
# high-water cost as later URLs may use more than the first one.
|
||||
# Never divide concurrent growth by task count: their pages vary.
|
||||
used, _available = observation
|
||||
# Only attribute this workload's growth to a capture. Other jobs on
|
||||
# the host reduce available headroom, but are not a per-capture cost
|
||||
# to multiply again for every snapshot we want to admit.
|
||||
self._memory_peak_cost = max(
|
||||
self._memory_peak_cost,
|
||||
used - baseline_used,
|
||||
baseline_host_available - host_available,
|
||||
used - self._memory_baseline,
|
||||
)
|
||||
if self._observed_snapshot_cost is not None:
|
||||
self._observed_snapshot_cost = max(self._observed_snapshot_cost, self._memory_peak_cost)
|
||||
@@ -615,6 +614,13 @@ class CrawlRunner:
|
||||
self._observe_snapshot_memory()
|
||||
if self._memory_peak_cost > 0:
|
||||
self._observed_snapshot_cost = self._memory_peak_cost
|
||||
headroom = resource_admission.memory_headroom()
|
||||
logging.getLogger(__name__).info(
|
||||
"Snapshot admission: concurrency limit=%s, observed memory growth=%s bytes, available=%s bytes",
|
||||
self.max_concurrent_snapshots,
|
||||
self._observed_snapshot_cost,
|
||||
headroom[1] if headroom else None,
|
||||
)
|
||||
else:
|
||||
# No observed cost cannot justify parallel admission. Probe the
|
||||
# next real snapshot while the queue keeps moving one at a time.
|
||||
@@ -716,6 +722,9 @@ class CrawlRunner:
|
||||
from archivebox.machine.models import Machine, NetworkInterface, Process
|
||||
|
||||
self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
|
||||
# Validate saved binary paths at the execution boundary, including
|
||||
# config edited since this process first registered its Machine.
|
||||
machine = Machine._sanitize_config(Machine.current())
|
||||
current_iface = NetworkInterface.current(refresh=not self.allow_maintenance_on_inactive_crawl)
|
||||
current_process = Process.current()
|
||||
if current_process.iface_id != current_iface.id or current_process.machine_id != current_iface.machine_id:
|
||||
@@ -724,7 +733,7 @@ class CrawlRunner:
|
||||
current_process.save(update_fields=["iface", "machine", "modified_at"])
|
||||
self.persona = self.crawl.resolve_persona()
|
||||
self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
|
||||
self.derived_config = dict(Machine.current().config or {})
|
||||
self.derived_config = dict(machine.config or {})
|
||||
self.crawl_output_dir = str(self.crawl.output_dir)
|
||||
if self.persona:
|
||||
self.base_config.update(
|
||||
|
||||
@@ -1527,6 +1527,9 @@
|
||||
display: inline-flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
box-sizing: border-box;
|
||||
gap: 4px;
|
||||
padding: 2px 4px;
|
||||
background: transparent;
|
||||
@@ -1540,15 +1543,27 @@
|
||||
flex-wrap: wrap;
|
||||
gap: 4px;
|
||||
align-items: center;
|
||||
min-width: 0;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.tag-editor-inline .tag-pill {
|
||||
min-width: 0;
|
||||
max-width: min(160px, 100%);
|
||||
box-sizing: border-box;
|
||||
padding: 2px 6px 2px 8px;
|
||||
font-size: 11px;
|
||||
border-radius: 12px;
|
||||
}
|
||||
|
||||
.tag-editor-inline .tag-link {
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.tag-editor-inline .tag-remove-btn {
|
||||
flex-shrink: 0;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
font-size: 12px;
|
||||
|
||||
@@ -79,6 +79,13 @@
|
||||
}
|
||||
.cards .card .card-tags span {
|
||||
display: inline-block;
|
||||
max-width: min(160px, 100%);
|
||||
box-sizing: border-box;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
vertical-align: middle;
|
||||
pointer-events: auto;
|
||||
padding: 2px 5px;
|
||||
border-radius: 5px;
|
||||
opacity: 0.95;
|
||||
@@ -283,16 +290,16 @@
|
||||
<div class="card-media">
|
||||
<a href="{% snapshot_detail_url obj %}" class="card-thumbnail">
|
||||
{% snapshot_thumbnail obj admin=True width="100%" height="245px" %}
|
||||
</a>
|
||||
{% if obj.tags_str %}
|
||||
<div class="card-tags">
|
||||
{% for tag in obj.tags_str|split:',' %}
|
||||
<span>
|
||||
<span title="{{tag}}">
|
||||
{{tag}}
|
||||
</span>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</a>
|
||||
</div>
|
||||
<div class="card-outputs">
|
||||
{% if obj.status == 'started' %}
|
||||
|
||||
@@ -198,21 +198,29 @@
|
||||
}
|
||||
|
||||
.snapshot-preview-cell {
|
||||
position: relative;
|
||||
width: 110px;
|
||||
min-width: 116px;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
.snapshot-preview-cell a {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
bottom: 10px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 102px;
|
||||
height: 102px;
|
||||
line-height: 0;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
.snapshot-preview-cell .snapshot-thumbnail {
|
||||
width: 100% !important;
|
||||
height: 100% !important;
|
||||
}
|
||||
|
||||
#content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
|
||||
#content .public-snapshot-list .snapshot-preview-cell img.snapshot-preview.screenshot {
|
||||
box-sizing: border-box;
|
||||
@@ -330,9 +338,10 @@
|
||||
}
|
||||
|
||||
.snapshot-tag {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
max-width: 160px;
|
||||
display: block;
|
||||
min-width: 0;
|
||||
max-width: min(160px, 100%);
|
||||
box-sizing: border-box;
|
||||
padding: 2px 7px;
|
||||
border: 1px solid #bfdbfe;
|
||||
border-radius: 999px;
|
||||
@@ -585,7 +594,8 @@
|
||||
|
||||
.snapshot-preview-cell a {
|
||||
width: 82px;
|
||||
height: 82px;
|
||||
top: 7px;
|
||||
bottom: 7px;
|
||||
}
|
||||
|
||||
#content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
|
||||
@@ -648,8 +658,8 @@
|
||||
{% if public_search_stream_pending %}
|
||||
<span class="archivebox-search-stream-status" role="status" aria-live="polite">Searching...</span>
|
||||
{% else %}
|
||||
{{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }}
|
||||
<span>(page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})</span>
|
||||
{{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }}{% endif %}
|
||||
<span>(page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %})</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
@@ -659,7 +669,7 @@
|
||||
<tr>
|
||||
<th class="snapshot-time">Saved</th>
|
||||
<th class="snapshot-preview-cell">Preview</th>
|
||||
<th>Snapshot ({{ page_obj.paginator.count }})</th>
|
||||
<th>Snapshot{% if public_snapshot_count != None %} ({{ public_snapshot_count }}){% endif %}</th>
|
||||
<th class="snapshot-tags-cell">Tags</th>
|
||||
<th class="snapshot-status-cell">Status</th>
|
||||
<th class="snapshot-files-cell">Files</th>
|
||||
@@ -687,7 +697,7 @@
|
||||
</table>
|
||||
</div>
|
||||
<div class="public-pagination">
|
||||
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
|
||||
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }} total{% endif %}
|
||||
<br>
|
||||
<span class="step-links">
|
||||
{% if page_obj.has_previous %}
|
||||
@@ -696,14 +706,14 @@
|
||||
{% endif %}
|
||||
|
||||
<span class="current">
|
||||
Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
|
||||
Page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %}
|
||||
</span>
|
||||
|
||||
{% if page_obj.has_next %}
|
||||
<a href="?{% url_replace page=page_obj.next_page_number %}">next</a>
|
||||
{% endif %}
|
||||
{% if page_obj.has_next %}
|
||||
<a href="?{% url_replace page=page_obj.paginator.num_pages %}">last »</a>
|
||||
{% if page_obj.has_next and public_page_count != None %}
|
||||
<a href="?{% url_replace page=public_page_count %}">last »</a>
|
||||
{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@@ -1671,6 +1671,13 @@
|
||||
return false
|
||||
}
|
||||
|
||||
// A new snapshot has only the synthetic "Browse all files" card.
|
||||
// Selecting it would replace the live monitor with an empty index.
|
||||
// Keep progress/empty state until the user explicitly opens a file.
|
||||
if (document.getElementById('main-frame-wrapper')?.dataset.hasOutputs !== '1') {
|
||||
return false
|
||||
}
|
||||
|
||||
const selectedCard = document.querySelector('.thumb-card.selected-card') || document.querySelector('.thumb-card')
|
||||
const selectedLink = selectedCard && (selectedCard.querySelector('a[target=preview]') || selectedCard.querySelector('a'))
|
||||
if (selectedCard && selectedLink) {
|
||||
|
||||
@@ -85,6 +85,7 @@
|
||||
link.href = '/admin/core/snapshot/?tags__id__exact=' + td.id;
|
||||
link.className = 'tag-link';
|
||||
link.textContent = td.name;
|
||||
link.title = td.name;
|
||||
pill.appendChild(link);
|
||||
|
||||
var removeBtn = document.createElement('button');
|
||||
|
||||
@@ -1155,7 +1155,19 @@ body.change-list:not(.model-snapshot) #changelist #toolbar,
|
||||
opacity: 0.75;
|
||||
}
|
||||
|
||||
#result_list tbody td.field-preview_icon {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
#result_list tbody td.field-preview_icon > a {
|
||||
position: absolute;
|
||||
inset: 4px 2px;
|
||||
display: block;
|
||||
}
|
||||
|
||||
#result_list tbody td.field-preview_icon .snapshot-thumbnail {
|
||||
width: 100% !important;
|
||||
height: 100% !important;
|
||||
transition: box-shadow 140ms ease, transform 140ms ease;
|
||||
}
|
||||
|
||||
@@ -2226,7 +2238,7 @@ tbody .output-link:hover {opacity: 1;}
|
||||
.model-snapshot.change-list #result_list tbody th.field-created_at_display > a,
|
||||
.model-snapshot.change-list #result_list tbody td.field-created_at_display > a {
|
||||
display: inline-flex;
|
||||
max-width: none;
|
||||
max-width: 100%;
|
||||
white-space: nowrap !important;
|
||||
}
|
||||
|
||||
@@ -2234,8 +2246,10 @@ tbody .output-link:hover {opacity: 1;}
|
||||
.model-snapshot.change-list #result_list tbody td.field-created_at_display .snapshot-created-at {
|
||||
display: inline-flex;
|
||||
flex-direction: row !important;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
gap: 5px;
|
||||
row-gap: 0;
|
||||
white-space: nowrap !important;
|
||||
}
|
||||
|
||||
@@ -2250,13 +2264,10 @@ tbody .output-link:hover {opacity: 1;}
|
||||
grid-area: preview;
|
||||
align-self: stretch;
|
||||
place-items: center;
|
||||
min-height: 110px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.model-snapshot.change-list #result_list tbody td.field-preview_icon .snapshot-thumbnail {
|
||||
width: 80px !important;
|
||||
height: 110px !important;
|
||||
max-width: 100%;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
@@ -2278,7 +2289,7 @@ tbody .output-link:hover {opacity: 1;}
|
||||
padding-left: 0;
|
||||
}
|
||||
|
||||
.model-snapshot.change-list #result_list tbody td.field-title_str a {
|
||||
.model-snapshot.change-list #result_list tbody td.field-title_str > a {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
overflow-wrap: anywhere;
|
||||
@@ -2514,26 +2525,12 @@ tbody .output-link:hover {opacity: 1;}
|
||||
}
|
||||
|
||||
.model-snapshot.change-list #result_list tbody tr {
|
||||
grid-template-columns: 24px 84px minmax(0, 1fr);
|
||||
gap: 2px 4px;
|
||||
grid-template-areas:
|
||||
"check preview date"
|
||||
"check preview title"
|
||||
"check preview tags"
|
||||
"check preview status"
|
||||
"check preview files"
|
||||
"check preview size";
|
||||
}
|
||||
|
||||
.model-snapshot.change-list #result_list tbody td.field-files .files-icons {
|
||||
gap: 0;
|
||||
}
|
||||
|
||||
.model-snapshot.change-list #result_list tbody td.field-status_with_progress,
|
||||
.model-snapshot.change-list #result_list tbody td.field-size_with_stats {
|
||||
justify-self: start;
|
||||
text-align: left;
|
||||
}
|
||||
}
|
||||
|
||||
body.embedded-change-list #content {
|
||||
|
||||
@@ -1663,6 +1663,8 @@ def resolve_abxpkg_binary_env(
|
||||
deps_from: Path | list[Path] | tuple[Path, ...] | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Resolve required test dependencies through abxpkg's normal install path."""
|
||||
from abxpkg.config import apply_exec_env
|
||||
|
||||
command_env = dict(env) if env is not None else os.environ.copy()
|
||||
command_env["ABXPKG_LIB_DIR"] = str(lib_dir)
|
||||
command = [
|
||||
@@ -1683,7 +1685,11 @@ def resolve_abxpkg_binary_env(
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
payload = json.loads(result.stdout)
|
||||
return {str(key): str(value) for key, value in payload.items()}
|
||||
# abxpkg env emits prepend/append deltas, not a replacement environment.
|
||||
# Applying a PATH ending in ':' verbatim hides the caller's abxpkg CLI.
|
||||
delta = {str(key): str(value) for key, value in payload.items()}
|
||||
apply_exec_env(delta, command_env)
|
||||
return {key: command_env[key] for key in delta}
|
||||
|
||||
|
||||
def run_test_hook(
|
||||
@@ -1781,7 +1787,7 @@ def install_real_binary(
|
||||
from archivebox.machine.models import Binary, Machine
|
||||
|
||||
binary = Binary.objects.create(
|
||||
machine=machine or Machine.current(refresh=True),
|
||||
machine=machine or Machine.current(),
|
||||
name=name,
|
||||
binproviders=binproviders,
|
||||
overrides=overrides or {},
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import pytest
|
||||
import os
|
||||
import time
|
||||
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
@@ -15,3 +17,19 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header
|
||||
response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/basic.txt", **api_headers)
|
||||
|
||||
assert response.status_code == 200, response.content
|
||||
|
||||
|
||||
def test_screencast_frame_is_checked_by_image_request(client, api_admin_user, api_headers):
|
||||
crawl = Crawl.objects.create(urls="https://example.com/screencast", created_by=api_admin_user)
|
||||
frame = crawl.output_dir / "chrome_screencast" / "latest.jpg"
|
||||
frame.parent.mkdir(parents=True, exist_ok=True)
|
||||
frame.write_bytes(b"stored frame")
|
||||
started_at = time.time()
|
||||
os.utime(frame, (started_at - 10, started_at - 10))
|
||||
url = f"/api/v1/crawls/crawl/{crawl.id}/files/chrome_screencast/latest.jpg?after={started_at}"
|
||||
assert client.get(url, **api_headers).status_code == 404
|
||||
os.utime(frame, (started_at + 1, started_at + 1))
|
||||
response = client.get(url, **api_headers)
|
||||
assert response.status_code == 200
|
||||
assert b"".join(response.streaming_content) == b"stored frame"
|
||||
assert client.get(url.replace(str(started_at), "invalid"), **api_headers).status_code == 400
|
||||
@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
machine = Machine.current(refresh=True)
|
||||
machine = Machine.current()
|
||||
binary = install_real_binary("python3", machine=machine)
|
||||
|
||||
response = client.get("/api/v1/machine/binaries", **api_headers)
|
||||
|
||||
@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
machine = Machine.current(refresh=True)
|
||||
machine = Machine.current()
|
||||
binary = install_real_binary("python3", machine=machine)
|
||||
|
||||
response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers)
|
||||
|
||||
@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
machine = Machine.current(refresh=True)
|
||||
machine = Machine.current()
|
||||
binary = install_real_binary("python3", machine=machine)
|
||||
|
||||
response = client.get("/api/v1/machine/binary/by-name/python3", **api_headers)
|
||||
|
||||
@@ -7,7 +7,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
machine = Machine.current(refresh=True)
|
||||
machine = Machine.current()
|
||||
|
||||
response = client.get(f"/api/v1/machine/machine/{machine.id}", **api_headers)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def test_basic_success_case_request(client, tmp_path, api_headers):
|
||||
Machine.current(refresh=True)
|
||||
Machine.current()
|
||||
|
||||
response = client.get("/api/v1/machine/machines", **api_headers)
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,377 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive tests for archivebox add command.
|
||||
Verify add creates snapshots in DB, crawls, source files, and archive directories.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import pty
|
||||
import select
|
||||
import time
|
||||
import termios
|
||||
import signal
|
||||
import subprocess
|
||||
|
||||
import psutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
cli_env,
|
||||
run_archivebox_cmd,
|
||||
)
|
||||
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
IMPORT_FORMAT_EXPECTATIONS = {
|
||||
"rss": {
|
||||
"url": "https://example.com/",
|
||||
"title": "RSS Example Import",
|
||||
"date": "2024-01-01",
|
||||
"tags": {"rss-tag", "metadata"},
|
||||
},
|
||||
"netscape": {
|
||||
"url": "https://www.iana.org/domains/reserved",
|
||||
"title": "IANA Reserved Domains",
|
||||
"date": "2024-01-02",
|
||||
"tags": {"netscape-tag", "metadata"},
|
||||
},
|
||||
"dom": {
|
||||
"url": "https://www.iana.org/help/example-domains",
|
||||
},
|
||||
"json": {
|
||||
"url": "https://example.com/?archivebox-json-import=1",
|
||||
"title": "JSON Import Example",
|
||||
"date": "2024-01-03",
|
||||
"tags": {"json-tag", "metadata"},
|
||||
},
|
||||
"jsonl": {
|
||||
"url": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"title": "JSONL Import Example",
|
||||
"date": "2024-01-04",
|
||||
"tags": {"jsonl-tag", "metadata"},
|
||||
},
|
||||
"txt": {
|
||||
"url": "https://example.org/",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_import_format_files(base_dir: Path) -> dict[str, Path]:
|
||||
files = {
|
||||
"rss": base_dir / "test_rss.xml",
|
||||
"netscape": base_dir / "test_netscape.html",
|
||||
"dom": base_dir / "test_dom.html",
|
||||
"json": base_dir / "test_bookmarks.json",
|
||||
"jsonl": base_dir / "test_bookmarks.jsonl",
|
||||
"txt": base_dir / "test_urls.txt",
|
||||
}
|
||||
files["rss"].write_text(
|
||||
"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>ArchiveBox RSS import fixture</title>
|
||||
<link>https://example.com/</link>
|
||||
<description>ArchiveBox RSS import fixture</description>
|
||||
<item>
|
||||
<title>RSS Example Import</title>
|
||||
<link>https://example.com/</link>
|
||||
<guid>https://example.com/</guid>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<category>rss-tag</category>
|
||||
<category>metadata</category>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["netscape"].write_text(
|
||||
"""<!DOCTYPE NETSCAPE-Bookmark-file-1>
|
||||
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
|
||||
<TITLE>Bookmarks</TITLE>
|
||||
<H1>Bookmarks</H1>
|
||||
<DL><p>
|
||||
<DT><A HREF="https://www.iana.org/domains/reserved" ADD_DATE="1704153600" TAGS="netscape-tag,metadata">IANA Reserved Domains</A>
|
||||
</DL><p>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["dom"].write_text(
|
||||
"""<!doctype html>
|
||||
<html>
|
||||
<head><title>DOM import fixture</title></head>
|
||||
<body>
|
||||
<a href="https://www.iana.org/help/example-domains">IANA Example Domains</a>
|
||||
</body>
|
||||
</html>
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["json"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"url": "https://example.com/?archivebox-json-import=1",
|
||||
"title": "JSON Import Example",
|
||||
"tags": ["json-tag", "metadata"],
|
||||
"bookmarked_at": "2024-01-03T00:00:00+00:00",
|
||||
},
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["jsonl"].write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"url": "https://example.com/?archivebox-jsonl-import=1",
|
||||
"title": "JSONL Import Example",
|
||||
"tags": "jsonl-tag,metadata",
|
||||
"bookmarked_at": "2024-01-04T00:00:00+00:00",
|
||||
},
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
files["txt"].write_text(
|
||||
"Plain text import fixture containing https://example.org/ as a real live URL.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return files
|
||||
|
||||
|
||||
IMPORT_FORMAT_ENV = {
|
||||
"USE_COLOR": "False",
|
||||
"SHOW_PROGRESS": "False",
|
||||
"PLUGINS": "parse_html_urls,parse_jsonl_urls,parse_netscape_urls,parse_rss_urls,parse_txt_urls",
|
||||
"SAVE_WGET": "False",
|
||||
"SAVE_HEADERS": "False",
|
||||
"USE_CHROME": "False",
|
||||
"URL_ALLOWLIST": r"example\.com|example\.org|iana\.org|www\.iana\.org",
|
||||
}
|
||||
|
||||
|
||||
def assert_expected_import_snapshots(cwd: Path, expected_urls: set[str]) -> None:
|
||||
allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
with use_archivebox_db(cwd):
|
||||
rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status"))
|
||||
counts = {url: 0 for url in expected_urls}
|
||||
bad_statuses = []
|
||||
for url, status in rows:
|
||||
counts[url] += 1
|
||||
if status not in allowed_statuses:
|
||||
bad_statuses.append((url, status))
|
||||
assert all(count == 1 for count in counts.values()), counts
|
||||
assert not bad_statuses, bad_statuses
|
||||
|
||||
|
||||
def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]:
|
||||
other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt"
|
||||
other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
|
||||
other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
|
||||
canary = tmp_path / "archivebox_shell_injection_canary"
|
||||
return (
|
||||
[
|
||||
safe_url,
|
||||
"file:///etc/hosts",
|
||||
"/etc/hosts",
|
||||
"../../../../etc/passwd",
|
||||
f"file://{other_crawl_source}",
|
||||
str(other_crawl_source),
|
||||
f"'; touch {canary}; #",
|
||||
f'" && touch {canary} && echo "',
|
||||
f"$(touch {canary})",
|
||||
f"`touch {canary}`",
|
||||
"""<?xml version="1.0"?>
|
||||
<!DOCTYPE rss [
|
||||
<!ENTITY localfile SYSTEM "file:///etc/hosts">
|
||||
]>
|
||||
<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
|
||||
<channel>
|
||||
<item><title>&localfile;</title><link>file:///etc/passwd</link></item>
|
||||
<xi:include href="file:///etc/hosts" parse="text"/>
|
||||
</channel>
|
||||
</rss>""",
|
||||
],
|
||||
canary,
|
||||
)
|
||||
|
||||
|
||||
def assert_no_file_or_shell_payload_snapshots(cwd: Path, *, canary: Path) -> None:
|
||||
with use_archivebox_db(cwd):
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
assert not canary.exists()
|
||||
assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
|
||||
for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
|
||||
assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"choice,plugins",
|
||||
[(choice, "chrome") for choice in ["skip", "retry", "abort", "ctrl-c", "noninteractive"]] + [("noninteractive", "archivewebpage")],
|
||||
)
|
||||
def test_add_interrupts_active_hook(initialized_archive, choice, plugins):
|
||||
env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True")
|
||||
# The timed observation below is for a running navigate hook. Resolve the
|
||||
# selected plugin's real binaries before starting it, so a cold install is
|
||||
# not counted as a hook-start failure.
|
||||
installed = run_archivebox_cmd(["install", plugins], cwd=initialized_archive, env=env, timeout=600)
|
||||
assert installed.returncode == 0, installed.stderr or installed.stdout
|
||||
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 160))
|
||||
output = bytearray()
|
||||
result = None
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def active_hook():
|
||||
with use_archivebox_db(initialized_archive):
|
||||
return Process.objects.filter(
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
|
||||
status="running",
|
||||
).first()
|
||||
|
||||
try:
|
||||
result = run_archivebox_cmd(
|
||||
["add", f"--plugins={plugins}", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL if choice == "noninteractive" else slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
read_until(lambda: active_hook() is not None)
|
||||
hook = active_hook()
|
||||
assert hook is not None
|
||||
result.send_signal(signal.SIGINT)
|
||||
if choice != "noninteractive":
|
||||
read_until(lambda: b"Choice [skip]:" in output)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
assert not psutil.pid_exists(hook.pid)
|
||||
if choice == "retry":
|
||||
os.write(master, b"r")
|
||||
read_until(lambda: (new_hook := active_hook()) is not None and new_hook.pid != hook.pid)
|
||||
prompt_offset = len(output)
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output[prompt_offset:])
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
if choice == "ctrl-c":
|
||||
result.send_signal(signal.SIGINT)
|
||||
else:
|
||||
os.write(master, b"a" if choice == "abort" else b"\r")
|
||||
read_until(lambda: result.poll() is not None)
|
||||
assert result.returncode == (130 if choice in {"abort", "ctrl-c", "noninteractive"} else 0), output.decode(errors="replace")
|
||||
if choice == "noninteractive":
|
||||
assert b"Choice [skip]:" not in output
|
||||
assert not psutil.pid_exists(hook.pid)
|
||||
assert b"Traceback" not in output
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert not ArchiveResult.objects.filter(hook_name="on_Snapshot__30_chrome_navigate").exists()
|
||||
if plugins == "archivewebpage":
|
||||
start = ArchiveResult.objects.get(hook_name="on_Snapshot__16_archivewebpage_start")
|
||||
assert start.status == ArchiveResult.StatusChoices.NORESULTS
|
||||
assert not ArchiveResult.objects.filter(hook_name="on_Snapshot__65_archivewebpage_stop").exists()
|
||||
assert not ArchiveResult.objects.filter(plugin="archivewebpage", status="succeeded").exists()
|
||||
assert not any(output["name"] == "archivewebpage" for output in start.snapshot.get_html_details_context()["archiveresults"])
|
||||
for log in (initialized_archive / "logs").glob("worker_runner_add_*.log"):
|
||||
assert "Choice [skip]:" not in log.read_text()
|
||||
finally:
|
||||
if result is not None and result.poll() is None:
|
||||
result.terminate()
|
||||
result.wait(timeout=15)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
|
||||
|
||||
def test_add_single_url_records_url_in_crawl(initialized_archive):
|
||||
"""Test that adding a single URL queues a crawl with the submitted URL."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "Crawl.save() outside runner process" not in result.stderr
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
|
||||
assert crawl.urls == "https://example.com"
|
||||
assert crawl.get_urls_list() == ["https://example.com"]
|
||||
assert snapshots == []
|
||||
|
||||
|
||||
def test_add_with_depth_0_flag(initialized_archive):
|
||||
"""Test that --depth=0 flag is accepted and works."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "unrecognized arguments: --depth" not in result.stderr
|
||||
|
||||
|
||||
def test_add_records_max_url_and_size_limits_on_crawl(initialized_archive):
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--index-only",
|
||||
"--depth=1",
|
||||
"--max-urls=3",
|
||||
"--crawl-max-size=45mb",
|
||||
"--crawl-timeout=120",
|
||||
"--snapshot-max-size=5mb",
|
||||
"https://example.com",
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
columns = {field.name for field in Crawl._meta.local_fields}
|
||||
with use_archivebox_db(initialized_archive):
|
||||
config = Crawl.objects.values_list("config", flat=True).get() or {}
|
||||
|
||||
assert {"max_urls", "crawl_max_size", "crawl_timeout", "snapshot_max_size"}.isdisjoint(columns)
|
||||
assert config["CRAWL_MAX_URLS"] == 3
|
||||
assert config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024
|
||||
assert config["CRAWL_TIMEOUT"] == 120
|
||||
assert config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024
|
||||
|
||||
|
||||
def test_add_without_args_shows_usage(initialized_archive):
|
||||
"""Test that add without URLs fails with a usage hint instead of crashing."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["add"],
|
||||
)
|
||||
|
||||
combined = result.stdout + result.stderr
|
||||
assert result.returncode != 0
|
||||
assert "usage" in combined.lower() or "url" in combined.lower()
|
||||
@@ -0,0 +1,422 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive tests for archivebox add command.
|
||||
Verify add creates snapshots in DB, crawls, source files, and archive directories.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import pty
|
||||
import select
|
||||
import time
|
||||
import termios
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.tests.conftest import (
|
||||
cli_env,
|
||||
run_archivebox_cmd,
|
||||
run_queued_crawls,
|
||||
)
|
||||
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
from .test_cli_add_1 import (
|
||||
pytestmark as pytestmark,
|
||||
IMPORT_FORMAT_EXPECTATIONS as IMPORT_FORMAT_EXPECTATIONS,
|
||||
write_import_format_files as write_import_format_files,
|
||||
IMPORT_FORMAT_ENV as IMPORT_FORMAT_ENV,
|
||||
assert_expected_import_snapshots as assert_expected_import_snapshots,
|
||||
malicious_add_inputs as malicious_add_inputs,
|
||||
assert_no_file_or_shell_payload_snapshots as assert_no_file_or_shell_payload_snapshots,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("terminal_stream", ["stdout", "stderr"])
|
||||
def test_add_renders_live_progress_in_foreground_terminal(initialized_archive, terminal_stream):
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 160))
|
||||
output = bytearray()
|
||||
result = None
|
||||
try:
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--plugins=parse_txt_urls", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=cli_env(USE_COLOR="True", SHOW_PROGRESS="True"),
|
||||
**{terminal_stream: slave},
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
elif result.poll() is not None:
|
||||
break
|
||||
assert result.poll() == 0, output.decode(errors="replace")
|
||||
assert b"\x1b[?25l" in output, output.decode(errors="replace")
|
||||
assert b"\x1b[?25h" in output, output.decode(errors="replace")
|
||||
assert b"on_Snapshot__71_parse_txt_urls" in output
|
||||
assert b"\x1b[2K" in output
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Crawl.objects.get().status == Crawl.StatusChoices.SEALED
|
||||
snapshot = Snapshot.objects.get()
|
||||
assert snapshot.url == "https://example.com"
|
||||
parsed_urls = Path(snapshot.output_dir) / "parse_txt_urls" / "urls.jsonl"
|
||||
assert json.loads(parsed_urls.read_text())["url"] == "https://iana.org/domains/example"
|
||||
finally:
|
||||
if result is not None and result.poll() is None:
|
||||
result.terminate()
|
||||
result.wait(timeout=10)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
|
||||
|
||||
def test_add_redirected_progress_remains_plain_text(initialized_archive):
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--plugins=parse_txt_urls", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=cli_env(),
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "on_Snapshot__71_parse_txt_urls" in result.stderr
|
||||
assert "\x1b[?25l" not in result.stdout + result.stderr
|
||||
|
||||
|
||||
@pytest.mark.timeout(360)
|
||||
def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initialized_archive):
|
||||
"""`archivebox add < import-file` should normalize rich import formats before crawling URLs."""
|
||||
import_files = write_import_format_files(initialized_archive)
|
||||
expected_urls = {case["url"] for case in IMPORT_FORMAT_EXPECTATIONS.values()}
|
||||
env = cli_env(**IMPORT_FORMAT_ENV)
|
||||
|
||||
for import_path in import_files.values():
|
||||
source_text = import_path.read_text(encoding="utf-8")
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--bg", "--depth=0", "--tag=cli-stdin-import"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=source_text,
|
||||
timeout=360,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.order_by("-created_at").first()
|
||||
assert crawl is not None
|
||||
assert crawl.snapshot_set.count() == 0
|
||||
assert crawl.urls == source_text
|
||||
|
||||
run_queued_crawls(initialized_archive, env=env, timeout=240)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for crawl in Crawl.objects.all():
|
||||
assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
|
||||
root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
|
||||
assert root_input == crawl.urls
|
||||
assert_expected_import_snapshots(initialized_archive, expected_urls)
|
||||
|
||||
list_result = run_archivebox_cmd(
|
||||
["list", "--json"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert list_result.returncode == 0, list_result.stderr or list_result.stdout
|
||||
for expected_url in expected_urls:
|
||||
assert expected_url in list_result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawls = list(Crawl.objects.order_by("created_at"))
|
||||
snapshots_by_url = {snapshot.url: snapshot for snapshot in Snapshot.objects.prefetch_related("tags").filter(url__in=expected_urls)}
|
||||
tags_by_url = {snapshot.url: set(snapshot.tags.values_list("name", flat=True)) for snapshot in snapshots_by_url.values()}
|
||||
|
||||
assert len(crawls) == len(import_files)
|
||||
assert [crawl.urls for crawl in crawls] == [path.read_text(encoding="utf-8") for path in import_files.values()]
|
||||
assert all(crawl.tags_str == "cli-stdin-import" for crawl in crawls)
|
||||
assert all(crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED} for crawl in crawls)
|
||||
assert len(snapshots_by_url) == len(expected_urls)
|
||||
|
||||
for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items():
|
||||
snapshot = snapshots_by_url.get(expected["url"])
|
||||
assert snapshot is not None, f"{import_name} did not create Snapshot for {expected['url']}"
|
||||
assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
if expected.get("title"):
|
||||
assert snapshot.title == expected["title"]
|
||||
if expected.get("date"):
|
||||
assert snapshot.bookmarked_at.date().isoformat() == expected["date"]
|
||||
if expected.get("tags"):
|
||||
assert expected["tags"] | {"cli-stdin-import"} <= tags_by_url[snapshot.url]
|
||||
|
||||
|
||||
def test_add_bg_queues_direct_url_snapshot(initialized_archive):
|
||||
"""Background add queues explicit URL arguments as real URL snapshots."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--bg", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
|
||||
assert crawl.status == Crawl.StatusChoices.QUEUED
|
||||
assert crawl.retry_at is not None
|
||||
assert crawl.urls == "https://example.com"
|
||||
assert snapshots == []
|
||||
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_add_title_hook_env_gets_canonical_runtime_config_without_archivebox_selectors_or_aliases(initialized_archive):
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "False",
|
||||
"SHOW_PROGRESS": "False",
|
||||
"PLUGINS": "title",
|
||||
"SAVE_TITLE": "True",
|
||||
"SECRET_KEY": "hook-env-secret-must-not-leak",
|
||||
"PUBLIC_ADD_VIEW": "True",
|
||||
"ADMIN_PASSWORD": "hook-env-admin-password-must-not-leak",
|
||||
"URL_BLACKLIST": "$^",
|
||||
"ARCHIVEBOX_TEST_ARBITRARY_ENV": "preserved-user-env",
|
||||
"LD_PRELOAD": "",
|
||||
"TIMEOUT": "60",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1",
|
||||
},
|
||||
)
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=0",
|
||||
"--plugins=title",
|
||||
"https://example.com/?archivebox-title-env=1",
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "Crawl.save() outside runner process" not in result.stdout + result.stderr
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshot = Snapshot.objects.get()
|
||||
title_result = ArchiveResult.objects.select_related("process").get(snapshot=snapshot, plugin="title")
|
||||
process_env = title_result.process.env
|
||||
|
||||
assert crawl.config["PLUGINS"] == "title"
|
||||
assert snapshot.url == "https://example.com/?archivebox-title-env=1"
|
||||
assert snapshot.status == Snapshot.StatusChoices.SEALED
|
||||
assert snapshot.title == "Example Domain"
|
||||
assert title_result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert title_result.output_str == "Example Domain"
|
||||
assert title_result.process.exit_code == 0
|
||||
assert process_env["TITLE_ENABLED"] == "True"
|
||||
assert process_env["CHROME_ENABLED"] == "True"
|
||||
assert process_env["WGET_ENABLED"] == "False"
|
||||
assert process_env["ARCHIVEBOX_TEST_ARBITRARY_ENV"] == "preserved-user-env"
|
||||
assert process_env["LD_PRELOAD"] == ""
|
||||
assert "PLUGINS" not in process_env
|
||||
assert "SAVE_TITLE" not in process_env
|
||||
assert "SEARCH_BACKEND_ENGINE" not in process_env
|
||||
assert "SECRET_KEY" not in process_env
|
||||
assert "ADMIN_PASSWORD" not in process_env
|
||||
assert "PUBLIC_ADD_VIEW" not in process_env
|
||||
assert "URL_BLACKLIST" not in process_env
|
||||
assert "hook-env-secret-must-not-leak" not in json.dumps(process_env)
|
||||
assert "hook-env-admin-password-must-not-leak" not in json.dumps(process_env)
|
||||
|
||||
|
||||
def test_add_creates_crawl_record(initialized_archive):
|
||||
"""Test that add command creates a Crawl record in the database."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl_count = Crawl.objects.count()
|
||||
|
||||
assert crawl_count == 1
|
||||
|
||||
|
||||
def test_add_rejects_file_path_argument(initialized_archive):
|
||||
"""Local files must be piped through stdin, not passed as archiveable path arguments."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
urls_file = initialized_archive / "urls.txt"
|
||||
urls_file.write_text("https://example.com\nhttps://example.org\n")
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", str(urls_file)],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Crawl.objects.count() == 0
|
||||
assert Snapshot.objects.count() == 0
|
||||
|
||||
assert "No URLs provided" in (result.stderr or result.stdout)
|
||||
|
||||
|
||||
def test_add_records_url_filter_overrides_on_crawl(initialized_archive):
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--index-only",
|
||||
"--depth=0",
|
||||
"--domain-allowlist=example.com,*.example.com",
|
||||
"--domain-denylist=static.example.com",
|
||||
"https://example.com",
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
|
||||
assert crawl.config["URL_ALLOWLIST"] == "example.com,*.example.com"
|
||||
assert crawl.config["URL_DENYLIST"] == "static.example.com"
|
||||
assert not (initialized_archive / "personas" / "Default" / "chrome_extensions").exists()
|
||||
|
||||
|
||||
def test_add_help_shows_depth_and_tag_options(initialized_archive):
|
||||
"""Test that add --help documents the main filter and crawl options."""
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--help"],
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "--depth" in result.stdout
|
||||
assert "--max-urls" in result.stdout
|
||||
assert "--crawl-max-size" in result.stdout
|
||||
assert "--crawl-timeout" in result.stdout
|
||||
assert "--snapshot-max-size" in result.stdout
|
||||
assert "--tag" in result.stdout
|
||||
|
||||
|
||||
def test_add_index_only_creates_direct_url_snapshot(initialized_archive):
|
||||
"""Index-only add seeds explicit URL args for the runner to snapshot."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshot = Snapshot.objects.get()
|
||||
|
||||
assert crawl.urls == "https://example.com"
|
||||
assert snapshot.url == "https://example.com"
|
||||
assert snapshot.depth == 0
|
||||
|
||||
|
||||
def test_snapshot_create_sets_snapshot_timestamp(initialized_archive):
|
||||
"""Test the user-facing snapshot creation path sets a timestamp."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
run_archivebox_cmd(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
check=True,
|
||||
)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
timestamp = Snapshot.objects.values_list("timestamp", flat=True).get()
|
||||
|
||||
assert timestamp is not None
|
||||
assert len(str(timestamp)) > 0
|
||||
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive, recursive_test_site):
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"TIMEOUT": "60",
|
||||
"SAVE_WGET": "true",
|
||||
"SAVE_HEADERS": "false",
|
||||
"SAVE_TITLE": "false",
|
||||
"SAVE_READABILITY": "false",
|
||||
"SAVE_SINGLEFILE": "false",
|
||||
"SAVE_MERCURY": "false",
|
||||
"SAVE_SCREENSHOT": "false",
|
||||
"SAVE_PDF": "false",
|
||||
"SAVE_DOM": "false",
|
||||
"SAVE_ARCHIVEDOTORG": "false",
|
||||
"SAVE_GIT": "false",
|
||||
"SAVE_YTDLP": "false",
|
||||
"SAVE_FAVICON": "false",
|
||||
"PARSE_HTML_URLS_ENABLED": "true",
|
||||
"PARSE_DOM_OUTLINKS_ENABLED": "false",
|
||||
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
|
||||
},
|
||||
)
|
||||
root_url = recursive_test_site["root_url"]
|
||||
child_url = recursive_test_site["child_urls"][0]
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=2",
|
||||
"--max-urls=2",
|
||||
"--crawl-max-size=50mb",
|
||||
"--tag=recursive-flow",
|
||||
"--parser=url_list",
|
||||
"--plugins=wget,parse_html_urls",
|
||||
root_url,
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()
|
||||
snapshots = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status"))
|
||||
archive_results = list(
|
||||
ArchiveResult.objects.select_related("snapshot")
|
||||
.order_by("snapshot__depth", "snapshot__url", "plugin")
|
||||
.values_list("snapshot__url", "plugin", "status", "output_files"),
|
||||
)
|
||||
|
||||
assert crawl[0] == 2
|
||||
assert crawl[1] == "recursive-flow"
|
||||
crawl_config = crawl[2] or {}
|
||||
assert crawl_config["CRAWL_MAX_URLS"] == 2
|
||||
assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024
|
||||
assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0
|
||||
assert (root_url, 0, "sealed") in snapshots
|
||||
assert any(url == child_url and depth == 1 and status == "sealed" for url, depth, status in snapshots)
|
||||
|
||||
by_url_plugin = {(url, plugin): status for url, plugin, status, _files in archive_results}
|
||||
assert by_url_plugin[(root_url, "wget")] == "succeeded"
|
||||
assert by_url_plugin[(root_url, "parse_html_urls")] == "succeeded"
|
||||
assert by_url_plugin[(child_url, "wget")] == "succeeded"
|
||||
|
||||
urls_outputs = list((initialized_archive / "archive/users/system/snapshots").rglob("parse_html_urls/urls.jsonl"))
|
||||
assert urls_outputs
|
||||
assert any(child_url in path.read_text() for path in urls_outputs)
|
||||
@@ -0,0 +1,456 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive tests for archivebox add command.
|
||||
Verify add creates snapshots in DB, crawls, source files, and archive directories.
|
||||
"""
|
||||
|
||||
import os
|
||||
import pty
|
||||
import select
|
||||
import time
|
||||
import termios
|
||||
import signal
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
import psutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
cli_env,
|
||||
run_archivebox_cmd,
|
||||
run_queued_crawls,
|
||||
resolve_abxpkg_chrome_env,
|
||||
)
|
||||
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
from .test_cli_add_1 import (
|
||||
pytestmark as pytestmark,
|
||||
IMPORT_FORMAT_EXPECTATIONS as IMPORT_FORMAT_EXPECTATIONS,
|
||||
write_import_format_files as write_import_format_files,
|
||||
IMPORT_FORMAT_ENV as IMPORT_FORMAT_ENV,
|
||||
assert_expected_import_snapshots as assert_expected_import_snapshots,
|
||||
malicious_add_inputs as malicious_add_inputs,
|
||||
assert_no_file_or_shell_payload_snapshots as assert_no_file_or_shell_payload_snapshots,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("choice", ["skip", "retry", "ctrl-c", "double", "noninteractive"])
|
||||
def test_add_interrupts_background_only_capture(initialized_archive, choice):
|
||||
"""Neither pause nor abort/reclaim semantics may depend on a foreground hook."""
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 200))
|
||||
output = bytearray()
|
||||
result = None
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.05)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def active_hook():
|
||||
with use_archivebox_db(initialized_archive):
|
||||
return Process.objects.filter(archiveresult__plugin="forumdl", status="running").order_by("-started_at").first()
|
||||
|
||||
try:
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--depth=1", "--plugins=forumdl", "https://news.ycombinator.com"],
|
||||
cwd=initialized_archive,
|
||||
env=cli_env(),
|
||||
stdin=subprocess.DEVNULL if choice == "noninteractive" else slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
read_until(lambda: active_hook() is not None)
|
||||
hook = active_hook()
|
||||
children = psutil.Process(result.pid).children(recursive=True)
|
||||
result.send_signal(signal.SIGINT)
|
||||
if choice == "double":
|
||||
read_until(lambda: b"Forwarding Ctrl+C" in output)
|
||||
result.send_signal(signal.SIGINT)
|
||||
elif choice != "noninteractive":
|
||||
read_until(lambda: b"Choice [skip]:" in output, timeout=20)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
assert not psutil.pid_exists(hook.pid)
|
||||
if choice == "retry":
|
||||
os.write(master, b"r")
|
||||
read_until(lambda: (next_hook := active_hook()) is not None and next_hook.pid != hook.pid)
|
||||
children.extend(psutil.Process(result.pid).children(recursive=True))
|
||||
offset = len(output)
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output[offset:], timeout=20)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
os.write(master, b"\r" if choice == "skip" else b"\x03")
|
||||
read_until(lambda: result.poll() is not None, timeout=30)
|
||||
assert result.returncode == (0 if choice == "skip" else 130), output.decode(errors="replace")
|
||||
assert not [child.pid for child in children if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
|
||||
assert not psutil.pid_exists(hook.pid)
|
||||
assert b"Traceback" not in output
|
||||
if choice == "noninteractive":
|
||||
assert b"Choice [skip]:" not in output
|
||||
# An explicit abort must not be mistaken for takeover and restart work.
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Process.objects.filter(worker_type="worker_runner", process_type="orchestrator").count() == 1
|
||||
if choice != "skip":
|
||||
assert b"Aborting crawl" in output
|
||||
finally:
|
||||
if result is not None and result.poll() is None:
|
||||
result.terminate()
|
||||
result.wait(timeout=20)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
|
||||
|
||||
def test_background_completions_render_above_interrupt_prompt(initialized_archive, recursive_test_site):
|
||||
env = cli_env(WGET_ARGS_EXTRA='["--limit-rate=20"]', INFINISCROLL_SCROLL_DELAY="10000", CHROME_HEADLESS="True")
|
||||
# The timed observation is for a running infinite-scroll hook. Resolve its
|
||||
# real browser and wget dependencies before the readiness window begins.
|
||||
installed = run_archivebox_cmd(
|
||||
["install", "wget", "chrome"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=600,
|
||||
)
|
||||
assert installed.returncode == 0, installed.stderr or installed.stdout
|
||||
env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env))
|
||||
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 200))
|
||||
output = bytearray()
|
||||
result = None
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
for log in sorted(initialized_archive.rglob("on_Snapshot__*.*.log")):
|
||||
print(f"\n{log.relative_to(initialized_archive)}:\n{log.read_text(errors='replace')}")
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def result_for(plugin):
|
||||
with use_archivebox_db(initialized_archive):
|
||||
return ArchiveResult.objects.filter(plugin=plugin).first()
|
||||
|
||||
try:
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--depth=1", "--plugins=wget,infiniscroll,parse_html_urls", recursive_test_site["root_url"]],
|
||||
cwd=initialized_archive,
|
||||
# The small test-owned page still transfers slowly enough for wget
|
||||
# to finish while the real foreground hook's prompt is open.
|
||||
env=env,
|
||||
stdin=slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
read_until(lambda: (scroll := result_for("infiniscroll")) is not None and scroll.status == "started")
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
before_completion = len(output)
|
||||
read_until(lambda: (wget := result_for("wget")) is not None and wget.status == "succeeded")
|
||||
read_until(lambda: b"Choice [skip]:" in output[before_completion:])
|
||||
screen_updates = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output[before_completion:].decode(errors="replace"))
|
||||
completed_at = screen_updates.index("on_Snapshot__35_wget")
|
||||
prompt_at = screen_updates.rindex("Interrupted on_Snapshot__45_infiniscroll")
|
||||
assert "succeeded" in screen_updates[completed_at:prompt_at], screen_updates
|
||||
assert completed_at < prompt_at < screen_updates.rindex("Choice [skip]:"), screen_updates
|
||||
assert result_for("parse_html_urls") is None
|
||||
(initialized_archive / "interrupt-terminal.txt").write_bytes(output)
|
||||
os.write(master, b"a")
|
||||
read_until(lambda: result.poll() is not None)
|
||||
assert result.returncode == 130, output.decode(errors="replace")
|
||||
assert b"Traceback" not in output
|
||||
finally:
|
||||
if result is not None and result.poll() is None:
|
||||
result.terminate()
|
||||
result.wait(timeout=15)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
|
||||
|
||||
def test_add_summary_points_to_snapshot_outputs(initialized_archive):
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--plugins=parse_txt_urls", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=cli_env(),
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot = Snapshot.objects.get()
|
||||
assert Path(snapshot.output_dir).is_dir()
|
||||
assert str(snapshot.output_dir) in result.stdout
|
||||
assert "snapshot output saved to:" in result.stdout
|
||||
assert "crawl logs and setup:" in result.stdout
|
||||
assert "crawl output saved to:" not in result.stdout
|
||||
|
||||
|
||||
def test_initial_crawl_creation_does_not_warn_as_an_outside_runner_update(initialized_archive, caplog):
|
||||
from archivebox.core.takeover_util import current_command
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
command = current_command(Process.TypeChoices.ADD, data_dir=initialized_archive)
|
||||
caplog.clear()
|
||||
with caplog.at_level("WARNING", logger="archivebox.workers.models"):
|
||||
crawl = Crawl.objects.create(urls="https://example.com", status=Crawl.StatusChoices.QUEUED)
|
||||
assert "Crawl.save() outside runner process" not in caplog.text
|
||||
|
||||
caplog.clear()
|
||||
with caplog.at_level("WARNING", logger="archivebox.workers.models"):
|
||||
crawl.save(update_fields=["modified_at"])
|
||||
command.mark_exited(exit_code=0)
|
||||
|
||||
assert "Crawl.save() outside runner process" in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive):
|
||||
"""CLI add must not turn user-supplied local paths or shell payloads into snapshots."""
|
||||
safe_url = "https://example.com/?archivebox-cli-security=1"
|
||||
inputs, canary = malicious_add_inputs(initialized_archive, safe_url=safe_url)
|
||||
env = cli_env(**IMPORT_FORMAT_ENV)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--bg", "--depth=0", "--tag=cli-security"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin="\n".join(inputs),
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
run_queued_crawls(initialized_archive, env=env, timeout=120)
|
||||
assert_expected_import_snapshots(initialized_archive, {safe_url})
|
||||
|
||||
assert_no_file_or_shell_payload_snapshots(initialized_archive, canary=canary)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshots = list(Snapshot.objects.filter(url=safe_url).values_list("url", "status", "tags__name"))
|
||||
crawl = Crawl.objects.get()
|
||||
assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
|
||||
assert len({url for url, _status, _tag in snapshots}) == 1
|
||||
assert all(
|
||||
status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
|
||||
for _url, status, _tag in snapshots
|
||||
)
|
||||
assert "cli-security" in {tag for _url, _status, tag in snapshots}
|
||||
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_run_rejects_depth_two_file_url_snapshot_injected_directly_with_db_update(initialized_archive):
|
||||
"""A queued Snapshot row with a file:// URL must not be allowed to run hooks or recurse."""
|
||||
secret_url = "https://example.com/?archivebox-sql-depth2-file-secret=1"
|
||||
local_source = initialized_archive / "not_owned_by_depth_two_snapshot.txt"
|
||||
local_source.write_text(f"{secret_url}\n", encoding="utf-8")
|
||||
file_url = local_source.resolve().as_uri()
|
||||
env = cli_env(**{**IMPORT_FORMAT_ENV, "PLUGINS": "parse_txt_urls", "SAVE_HEADERS": "False", "SAVE_WGET": "False"})
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com/?archivebox-sql-depth2-root=1",
|
||||
max_depth=2,
|
||||
tags_str="sql-depth2-file-url",
|
||||
status=Crawl.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
root_snapshot = Snapshot.objects.create(
|
||||
url="https://example.com/?archivebox-sql-depth2-root=1",
|
||||
crawl=crawl,
|
||||
depth=0,
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
retry_at=None,
|
||||
)
|
||||
injected_snapshot = Snapshot.objects.create(
|
||||
url="https://example.com/?archivebox-sql-depth2-placeholder=1",
|
||||
crawl=crawl,
|
||||
parent_snapshot=root_snapshot,
|
||||
depth=2,
|
||||
status=Snapshot.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
Snapshot.objects.filter(pk=injected_snapshot.pk).update(url=file_url)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["run", f"--snapshot-id={injected_snapshot.id}"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
injected_snapshot.refresh_from_db()
|
||||
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
|
||||
file_results = list(
|
||||
ArchiveResult.objects.filter(
|
||||
snapshot=injected_snapshot,
|
||||
).values_list("plugin", "status"),
|
||||
)
|
||||
|
||||
assert injected_snapshot.url == file_url
|
||||
assert secret_url not in snapshot_urls
|
||||
assert file_results == []
|
||||
|
||||
|
||||
def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initialized_archive):
|
||||
"""Index-only add only creates the crawl; rejected URLs are sealed by the runner."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--index-only",
|
||||
"--depth=0",
|
||||
"--url-denylist=example.com",
|
||||
"https://example.com",
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
|
||||
|
||||
assert crawl.status == Crawl.StatusChoices.QUEUED
|
||||
assert crawl.retry_at is None
|
||||
assert crawl.urls == "https://example.com"
|
||||
assert snapshot_urls == set()
|
||||
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
|
||||
|
||||
assert crawl.status == Crawl.StatusChoices.SEALED
|
||||
assert crawl.retry_at is None
|
||||
assert crawl.urls == "https://example.com"
|
||||
assert snapshot_urls == set()
|
||||
|
||||
|
||||
def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
|
||||
"""Index-only add must apply the same internal URL guard as snapshot creation."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
internal_urls = [
|
||||
"http://archivebox.localhost:9292/admin/",
|
||||
"http://web.archivebox.localhost:9292/",
|
||||
"http://api.archivebox.localhost:9292/api/v1/docs",
|
||||
"http://snap-2fb8e923c58c.archivebox.localhost:9292/index.html",
|
||||
]
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", *internal_urls],
|
||||
cwd=initialized_archive,
|
||||
env={**env, "BASE_URL": "http://archivebox.localhost:9292"},
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
|
||||
|
||||
assert crawl.urls.splitlines() == internal_urls
|
||||
assert crawl.status == Crawl.StatusChoices.QUEUED
|
||||
assert crawl.retry_at is None
|
||||
assert snapshot_urls == set()
|
||||
|
||||
|
||||
def test_add_multiple_urls_single_command(initialized_archive):
|
||||
"""Test adding multiple URLs in a single command records one crawl."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com", "https://example.org"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth"))
|
||||
|
||||
assert crawl.urls.splitlines() == ["https://example.com", "https://example.org"]
|
||||
assert snapshots == []
|
||||
|
||||
|
||||
def test_add_with_tags(initialized_archive):
|
||||
"""Test adding URL with tags stores tags_str in crawl.
|
||||
|
||||
With --index-only, Tag objects are not created until archiving happens.
|
||||
Tags are stored as a string in the Crawl.tags_str field.
|
||||
"""
|
||||
env = cli_env(disable_extractors=True)
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "--tag=test,example", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
tags_str = Crawl.objects.values_list("tags_str", flat=True).get()
|
||||
|
||||
# Tags are stored as a comma-separated string in crawl
|
||||
assert "test" in tags_str or "example" in tags_str
|
||||
|
||||
|
||||
def test_add_with_overwrite_flag(initialized_archive):
|
||||
"""Test that --overwrite flag forces re-archiving."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
# Add URL first time
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
# Add with overwrite
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--overwrite", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "unrecognized arguments: --overwrite" not in result.stderr
|
||||
|
||||
|
||||
def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive):
|
||||
"""Test that --index-only creates only a queued crawl and returns fast."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=30, # Should be fast
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshots = list(Snapshot.objects.all())
|
||||
|
||||
assert crawl.status == Crawl.StatusChoices.QUEUED
|
||||
assert crawl.retry_at is None
|
||||
assert crawl.urls == "https://example.com"
|
||||
assert snapshots == []
|
||||
@@ -0,0 +1,511 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive tests for archivebox add command.
|
||||
Verify add creates snapshots in DB, crawls, source files, and archive directories.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import pty
|
||||
import select
|
||||
import time
|
||||
import termios
|
||||
import signal
|
||||
import re
|
||||
|
||||
import psutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
cli_env,
|
||||
find_snapshot_dir,
|
||||
run_archivebox_cmd,
|
||||
run_queued_crawls,
|
||||
resolve_abxpkg_chrome_env,
|
||||
)
|
||||
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
from .test_cli_add_1 import (
|
||||
pytestmark as pytestmark,
|
||||
IMPORT_FORMAT_EXPECTATIONS as IMPORT_FORMAT_EXPECTATIONS,
|
||||
write_import_format_files as write_import_format_files,
|
||||
IMPORT_FORMAT_ENV as IMPORT_FORMAT_ENV,
|
||||
assert_expected_import_snapshots as assert_expected_import_snapshots,
|
||||
malicious_add_inputs as malicious_add_inputs,
|
||||
assert_no_file_or_shell_payload_snapshots as assert_no_file_or_shell_payload_snapshots,
|
||||
)
|
||||
|
||||
|
||||
def test_abort_stops_long_running_background_hook(initialized_archive):
|
||||
env = cli_env(INFINISCROLL_SCROLL_DELAY="10000", CHROME_HEADLESS="True")
|
||||
# Keep first-use dependency installation outside the timed hook-start
|
||||
# observation. This test is about interrupting a running hook; a cold
|
||||
# browser or forumdl install must not consume its readiness window.
|
||||
installed = run_archivebox_cmd(
|
||||
["install", "forumdl", "chrome"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=600,
|
||||
)
|
||||
assert installed.returncode == 0, installed.stderr or installed.stdout
|
||||
env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env))
|
||||
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 200))
|
||||
output = bytearray()
|
||||
result = None
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def result_for(plugin):
|
||||
with use_archivebox_db(initialized_archive):
|
||||
return ArchiveResult.objects.filter(plugin=plugin).first()
|
||||
|
||||
try:
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--depth=1", "--plugins=forumdl,infiniscroll", "https://news.ycombinator.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
read_until(lambda: (scroll := result_for("infiniscroll")) is not None and scroll.status == "started")
|
||||
except AssertionError:
|
||||
# Capture state before teardown terminates the runner: otherwise a
|
||||
# missed readiness deadline loses the install/hook failure itself.
|
||||
with use_archivebox_db(initialized_archive):
|
||||
print("Hook readiness:", list(ArchiveResult.objects.values("plugin", "status", "output_str")))
|
||||
print("Process readiness:", list(Process.objects.values("process_type", "status", "exit_code", "stderr")))
|
||||
raise
|
||||
read_until(lambda: "running pid=" in re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output.decode(errors="replace")))
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
forum = result_for("forumdl")
|
||||
assert forum is not None and forum.status == "started"
|
||||
children = psutil.Process(result.pid).children(recursive=True)
|
||||
abort_offset = len(output)
|
||||
os.write(master, b"\x03")
|
||||
read_until(lambda: b"Aborting crawl" in output[abort_offset:], timeout=3)
|
||||
read_until(lambda: result.poll() is not None, timeout=25)
|
||||
assert result.returncode == 130, output.decode(errors="replace")
|
||||
assert not [child.pid for child in children if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
|
||||
assert any("INFO:root:GET" in log.read_text() for log in initialized_archive.rglob("on_Snapshot__33_forumdl.*.stderr.log"))
|
||||
assert b"Stopped during crawl abort" in output
|
||||
assert b"INFO:root:GET" not in output[abort_offset:]
|
||||
assert b"Traceback" not in output
|
||||
finally:
|
||||
if result is not None and result.poll() is None:
|
||||
result.terminate()
|
||||
result.wait(timeout=15)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
|
||||
|
||||
def test_add_bg_persists_queue_without_starting_extraction_workers(initialized_archive):
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--bg", "--plugins=wget", "--crawl-max-concurrent-snapshots=4", "https://example.com", "https://example.org"],
|
||||
cwd=initialized_archive,
|
||||
env=cli_env(),
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
assert crawl.status == Crawl.StatusChoices.QUEUED
|
||||
assert crawl.retry_at is not None
|
||||
assert crawl.get_urls_list() == ["https://example.com", "https://example.org"]
|
||||
assert crawl.config["PLUGINS"] == "wget"
|
||||
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 4
|
||||
assert not Snapshot.objects.exists()
|
||||
assert not ArchiveResult.objects.exists()
|
||||
assert not Process.objects.filter(process_type__in=[Process.TypeChoices.ORCHESTRATOR, Process.TypeChoices.HOOK]).exists()
|
||||
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_run_rejects_file_url_injected_directly_into_crawl_urls_with_db_update(initialized_archive):
|
||||
"""Runner must validate Crawl.urls again when DB writes bypass normal add/create paths."""
|
||||
secret_url = "https://example.com/?archivebox-sql-crawl-file-secret=1"
|
||||
local_source = initialized_archive / "not_owned_by_crawl_urls.txt"
|
||||
local_source.write_text(f"{secret_url}\n", encoding="utf-8")
|
||||
file_url = local_source.resolve().as_uri()
|
||||
env = cli_env(**{**IMPORT_FORMAT_ENV, "PLUGINS": "parse_txt_urls", "SAVE_HEADERS": "False", "SAVE_WGET": "False"})
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com/?archivebox-sql-crawl-control=1",
|
||||
max_depth=2,
|
||||
tags_str="sql-file-url",
|
||||
status=Crawl.StatusChoices.QUEUED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
bad_jsonl = json.dumps({"type": "Snapshot", "url": file_url, "depth": 0, "tags": "sql-file-url"})
|
||||
Crawl.objects.filter(pk=crawl.pk).update(urls=bad_jsonl)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["run", f"--crawl-id={crawl.id}"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl.refresh_from_db()
|
||||
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
|
||||
|
||||
assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
|
||||
assert file_url not in snapshot_urls
|
||||
assert secret_url not in snapshot_urls
|
||||
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_add_tagged_single_url_seals_without_duplicate_snapshot_tags(initialized_archive):
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "False",
|
||||
"SHOW_PROGRESS": "False",
|
||||
"PLUGINS": "parse_txt_urls,title",
|
||||
"TIMEOUT": "60",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1",
|
||||
},
|
||||
)
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--bg",
|
||||
"--depth=0",
|
||||
"--tag=tagged-single-url",
|
||||
"https://example.com/?archivebox-tagged-single-url=1",
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
run_queued_crawls(initialized_archive, env, timeout=180)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
snapshots = list(Snapshot.objects.order_by("depth", "url"))
|
||||
tag_counts = {
|
||||
snapshot.url: SnapshotTag.objects.filter(snapshot=snapshot, tag__name="tagged-single-url").count() for snapshot in snapshots
|
||||
}
|
||||
results = list(
|
||||
ArchiveResult.objects.select_related("snapshot")
|
||||
.order_by("snapshot__depth", "snapshot__url", "plugin")
|
||||
.values_list("snapshot__url", "plugin", "status", "output_str"),
|
||||
)
|
||||
|
||||
assert crawl.status == Crawl.StatusChoices.SEALED
|
||||
assert [(snapshot.url, snapshot.depth, snapshot.status) for snapshot in snapshots] == [
|
||||
("https://example.com/?archivebox-tagged-single-url=1", 0, Snapshot.StatusChoices.SEALED),
|
||||
]
|
||||
assert tag_counts == {
|
||||
"https://example.com/?archivebox-tagged-single-url=1": 1,
|
||||
}
|
||||
by_url_plugin = {(url, plugin): status for url, plugin, status, _output in results}
|
||||
assert by_url_plugin[("https://example.com/?archivebox-tagged-single-url=1", "title")] == "succeeded"
|
||||
unexpected_failures = [(url, plugin, status, output) for url, plugin, status, output in results if status == "failed"]
|
||||
assert not unexpected_failures
|
||||
|
||||
|
||||
def test_add_direct_url_creates_snapshot_without_internal_input_file(initialized_archive):
|
||||
"""Explicit URL args materialize real snapshots when the runner claims the crawl."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot = Snapshot.objects.get()
|
||||
assert snapshot.url == "https://example.com"
|
||||
assert snapshot.depth == 0
|
||||
assert not (snapshot.output_dir / "staticfile" / "stdin.txt").exists()
|
||||
|
||||
|
||||
def test_add_with_depth_1_flag(initialized_archive):
|
||||
"""Test that --depth=1 flag is accepted."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=1", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "unrecognized arguments: --depth" not in result.stderr
|
||||
|
||||
|
||||
def test_add_rejects_invalid_depth_values(initialized_archive):
|
||||
"""Test that add rejects depth values outside the supported range."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
|
||||
for depth in ("5", "-1"):
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", f"--depth={depth}", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
stderr = result.stderr.lower()
|
||||
assert result.returncode != 0
|
||||
assert "invalid" in stderr or "not one of" in stderr
|
||||
|
||||
|
||||
def test_add_records_selected_persona_on_crawl(initialized_archive):
|
||||
"""Test add persists the selected persona so browser config derives from it later."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "--persona=Default", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get()
|
||||
|
||||
assert crawl.persona_id
|
||||
assert "ACTIVE_PERSONA" not in crawl.config
|
||||
assert (initialized_archive / "personas" / "Default" / "chrome_profile").is_dir()
|
||||
|
||||
|
||||
def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
|
||||
"""Test that adding the same URL twice creates separate crawls.
|
||||
|
||||
Each 'add' command creates a new Crawl. Multiple crawls can archive the same URL.
|
||||
This allows re-archiving URLs at different times.
|
||||
"""
|
||||
|
||||
env = cli_env(disable_extractors=True)
|
||||
# Add URL first time
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
|
||||
# Add same URL second time with --update to opt out of ONLY_NEW.
|
||||
run_archivebox_cmd(
|
||||
["add", "--index-only", "--update", "--depth=0", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl_count = Crawl.objects.count()
|
||||
snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "depth"))
|
||||
|
||||
# Each add creates a new crawl with its own queued work.
|
||||
assert crawl_count == 2
|
||||
assert snapshots == [("https://example.com", 0), ("https://example.com", 0)]
|
||||
|
||||
|
||||
def test_snapshot_create_creates_current_output_directory(initialized_archive):
|
||||
"""Test the user-facing snapshot creation path creates an output directory."""
|
||||
env = cli_env(disable_extractors=True)
|
||||
run_archivebox_cmd(
|
||||
["snapshot", "create", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
check=True,
|
||||
)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get())
|
||||
|
||||
snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
|
||||
assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
|
||||
assert snapshot_dir.is_dir()
|
||||
|
||||
|
||||
def test_add_preserves_apostrophe_in_snapshot_url(initialized_archive):
|
||||
url = "https://aaib.gov.in/What's%20New%20Assets/Preliminary%20Report%20VT-EXO.pdf"
|
||||
env = cli_env(disable_extractors=True)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--index-only", "--depth=0", url],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
run_queued_crawls(initialized_archive, env)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Crawl.objects.get().urls == url
|
||||
assert Snapshot.objects.get().url == url
|
||||
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_cli_add_real_urls_with_options_writes_inspectable_outputs(initialized_archive):
|
||||
|
||||
wget_urls = [
|
||||
"https://example.com",
|
||||
"https://pirate.github.io/stress-tests/challenge.html",
|
||||
]
|
||||
chrome_url = "https://example.com/?archivebox-chrome-flow=1"
|
||||
env = os.environ.copy()
|
||||
env.pop("CHROME_BINARY", None)
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"TIMEOUT": "60",
|
||||
"SAVE_WGET": "true",
|
||||
"SAVE_HEADERS": "false",
|
||||
"SAVE_TITLE": "false",
|
||||
"SAVE_READABILITY": "false",
|
||||
"SAVE_SINGLEFILE": "false",
|
||||
"SAVE_MERCURY": "false",
|
||||
"SAVE_SCREENSHOT": "false",
|
||||
"SAVE_PDF": "false",
|
||||
"SAVE_DOM": "false",
|
||||
"SAVE_ARCHIVEDOTORG": "false",
|
||||
"SAVE_GIT": "false",
|
||||
"SAVE_YTDLP": "false",
|
||||
"SAVE_FAVICON": "false",
|
||||
},
|
||||
)
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=0",
|
||||
"--max-urls=2",
|
||||
"--crawl-max-size=10mb",
|
||||
"--tag=real-flow,challenge",
|
||||
"--parser=url_list",
|
||||
"--plugins=wget",
|
||||
*wget_urls,
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert returncode == 0, stderr or stdout
|
||||
|
||||
chrome_env = env | {
|
||||
"SAVE_WGET": "false",
|
||||
"SAVE_HEADERS": "true",
|
||||
"SAVE_TITLE": "true",
|
||||
"CHROME_HEADLESS": "true",
|
||||
"CHROME_SANDBOX": "false",
|
||||
"CHROME_ISOLATION": "snapshot",
|
||||
}
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["install", "chrome"],
|
||||
cwd=initialized_archive,
|
||||
env=chrome_env,
|
||||
timeout=600,
|
||||
)
|
||||
install_stdout, install_stderr, install_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert install_returncode == 0, install_stderr or install_stdout
|
||||
chrome_env.update(resolve_abxpkg_chrome_env(Path(chrome_env["ABXPKG_LIB_DIR"]), chrome_env))
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=0",
|
||||
"--max-urls=1",
|
||||
"--crawl-max-size=10mb",
|
||||
"--tag=chrome-flow",
|
||||
"--parser=url_list",
|
||||
"--plugins=chrome,wget,headers,title",
|
||||
chrome_url,
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=chrome_env,
|
||||
timeout=180,
|
||||
)
|
||||
chrome_stdout, chrome_stderr, chrome_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert chrome_returncode == 0, chrome_stderr or chrome_stdout
|
||||
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["list", "--tag=real-flow"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
list_stdout, list_stderr, list_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert list_returncode == 0, list_stderr or list_stdout
|
||||
listed = [json.loads(line) for line in list_stdout.splitlines() if line.strip()]
|
||||
assert {item["url"] for item in listed} >= set(wget_urls)
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()
|
||||
real_flow_crawl = Crawl.objects.filter(tags_str="real-flow,challenge").values_list("max_depth", "tags_str", "config").first()
|
||||
snapshots = list(Snapshot.objects.order_by("url").values_list("id", "url", "depth", "status", "title"))
|
||||
archive_results = list(
|
||||
ArchiveResult.objects.select_related("snapshot")
|
||||
.order_by("snapshot__url", "plugin")
|
||||
.values_list("snapshot__url", "plugin", "status", "output_files", "output_size", "output_str"),
|
||||
)
|
||||
processes = list(Process.objects.filter(process_type="hook").values_list("process_type", "status", "exit_code", "pwd", "cmd"))
|
||||
|
||||
assert real_flow_crawl is not None
|
||||
assert real_flow_crawl[0] == 0
|
||||
assert real_flow_crawl[1] == "real-flow,challenge"
|
||||
real_flow_config = real_flow_crawl[2] or {}
|
||||
assert real_flow_config["CRAWL_MAX_URLS"] == 2
|
||||
assert real_flow_config["CRAWL_MAX_SIZE"] == 10 * 1024 * 1024
|
||||
assert real_flow_config.get("SNAPSHOT_MAX_SIZE", 0) == 0
|
||||
assert "wget" in real_flow_config["PLUGINS"]
|
||||
assert crawl is not None
|
||||
assert crawl[1] == "chrome-flow"
|
||||
assert "wget,headers,title" in json.dumps(crawl[2] or {})
|
||||
|
||||
snapshot_urls = {url for _id, url, _depth, _status, _title in snapshots}
|
||||
assert snapshot_urls >= {*wget_urls, chrome_url}
|
||||
assert all(depth == 0 for _id, url, depth, _status, _title in snapshots)
|
||||
|
||||
by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size, _output in archive_results}
|
||||
assert by_url_plugin[("https://example.com", "wget")] == "succeeded"
|
||||
assert by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", "wget")] == "succeeded"
|
||||
assert by_url_plugin[(chrome_url, "headers")] == "succeeded"
|
||||
assert by_url_plugin[(chrome_url, "title")] == "succeeded"
|
||||
unexpected_results = [
|
||||
(url, plugin, status, output) for url, plugin, status, _files, _size, output in archive_results if status != "succeeded"
|
||||
]
|
||||
assert not unexpected_results
|
||||
|
||||
snapshot_root = initialized_archive / "archive/users/system/snapshots"
|
||||
html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()]
|
||||
header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0]
|
||||
title_outputs = [path for path in snapshot_root.rglob("title/title.txt") if path.is_file() and path.stat().st_size > 0]
|
||||
index_outputs = [path for path in snapshot_root.rglob("index.jsonl") if path.is_file()]
|
||||
assert html_outputs
|
||||
assert header_outputs
|
||||
assert any("example.com" in path.read_text(errors="ignore").lower() for path in header_outputs)
|
||||
assert title_outputs
|
||||
assert any("Example Domain" in path.read_text(errors="ignore") for path in title_outputs)
|
||||
assert len(index_outputs) >= len(wget_urls) + 1
|
||||
|
||||
combined_html = "\n".join(path.read_text(errors="ignore") for path in html_outputs)
|
||||
assert "Example Domain" in combined_html
|
||||
assert "Browser Agent Challenge for AI Browser Drivers" in combined_html
|
||||
|
||||
assert processes
|
||||
assert any("wget" in (pwd or "") or "wget" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
|
||||
assert any("headers" in (pwd or "") or "headers" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
|
||||
@@ -269,6 +269,7 @@ def test_init_recovers_from_pre_squash_dev_history(tmp_path):
|
||||
("core", "0021_auto_20220724_1254"),
|
||||
("core", "0021_auto_20221128_1116"),
|
||||
("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
|
||||
("core", "0056_progress_covering_indexes"),
|
||||
("core", "0074_alter_snapshot_downloaded_at"),
|
||||
("core", "0075_crawl"),
|
||||
("core", "0075_archiveresult_retry_at"),
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,177 @@
|
||||
"""
|
||||
Tests for archivebox run CLI command.
|
||||
|
||||
Tests cover:
|
||||
- run with stdin JSONL (Crawl, Snapshot, ArchiveResult)
|
||||
- create-or-update behavior (records with/without id)
|
||||
- pass-through output (for chaining)
|
||||
"""
|
||||
|
||||
import os
|
||||
import pty
|
||||
import select
|
||||
import signal
|
||||
import subprocess
|
||||
import termios
|
||||
import time
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
from archivebox.tests.conftest import (
|
||||
cli_env,
|
||||
create_test_url,
|
||||
parse_jsonl_output,
|
||||
run_archivebox_cmd,
|
||||
)
|
||||
|
||||
RUN_TEST_ENV = {
|
||||
"PLUGINS": "favicon",
|
||||
"SAVE_FAVICON": "True",
|
||||
}
|
||||
|
||||
|
||||
def _install_real_chrome_for_test(data_dir, env, *, isolation):
|
||||
env["CHROME_ISOLATION"] = isolation
|
||||
env["CHROME_HEADLESS"] = "true"
|
||||
env["CHROME_SANDBOX"] = "false"
|
||||
install_process = run_archivebox_cmd(
|
||||
["install", "chrome"],
|
||||
cwd=data_dir,
|
||||
env=env,
|
||||
timeout=600,
|
||||
)
|
||||
assert install_process.returncode == 0, install_process.stderr or install_process.stdout
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.parametrize(
|
||||
"choice,target",
|
||||
[(choice, "crawl") for choice in ["skip", "retry", "abort", "ctrl-c", "noninteractive"]]
|
||||
+ [("skip", "snapshot"), ("skip", "global"), ("early-skip", "crawl")]
|
||||
+ [(choice, "daemon") for choice in ["daemon-tty", "daemon-noninteractive"]]
|
||||
+ [("daemon-term", "daemon")],
|
||||
)
|
||||
def test_run_crawl_interrupts_active_hook(initialized_archive, choice, target):
|
||||
"""Foreground run uses the same terminal interrupt choices as add."""
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True", PLUGINS="chrome")
|
||||
# This test starts its 45-second observation at the active navigate hook.
|
||||
# Install Chrome through the real CLI first so a cold binary download is
|
||||
# not mistaken for a hook that never started.
|
||||
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], cwd=initialized_archive, env=env)
|
||||
assert created.returncode == 0, created.stdout + created.stderr
|
||||
snapshot = next(record for record in parse_jsonl_output(created.stdout) if record.get("type") == "Snapshot")
|
||||
crawl_id = snapshot["crawl_id"]
|
||||
run_args = {
|
||||
"crawl": [f"--crawl-id={crawl_id}"],
|
||||
"snapshot": [f"--snapshot-id={snapshot['id']}"],
|
||||
"global": [],
|
||||
"daemon": ["--daemon", f"--crawl-id={crawl_id}"],
|
||||
}[target]
|
||||
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 160))
|
||||
output = bytearray()
|
||||
result = None
|
||||
owned_processes = []
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def active_hook():
|
||||
with use_archivebox_db(initialized_archive):
|
||||
return Process.objects.filter(
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
|
||||
status="running",
|
||||
).first()
|
||||
|
||||
try:
|
||||
result = run_archivebox_cmd(
|
||||
["run", *run_args],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL if choice in {"noninteractive", "daemon-noninteractive"} else slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
read_until(lambda: active_hook() is not None)
|
||||
first_hook = active_hook()
|
||||
assert first_hook is not None
|
||||
with use_archivebox_db(initialized_archive):
|
||||
hook_pids = Process.objects.filter(status="running", process_type="hook").values_list("pid", flat=True)
|
||||
for pid in hook_pids:
|
||||
try:
|
||||
process = psutil.Process(pid)
|
||||
owned_processes.extend([process, *process.children(recursive=True)])
|
||||
except psutil.NoSuchProcess:
|
||||
continue
|
||||
result.send_signal(signal.SIGTERM if choice == "daemon-term" else signal.SIGINT)
|
||||
if choice not in {"noninteractive", "daemon-tty", "daemon-noninteractive", "daemon-term"}:
|
||||
read_until(lambda: b"Choice [skip]:" in output, timeout=30)
|
||||
if choice != "early-skip":
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
if choice == "retry":
|
||||
os.write(master, b"r")
|
||||
read_until(lambda: (next_hook := active_hook()) is not None and next_hook.pid != first_hook.pid)
|
||||
before_prompt = len(output)
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output[before_prompt:], timeout=30)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
if choice == "ctrl-c":
|
||||
result.send_signal(signal.SIGINT)
|
||||
else:
|
||||
os.write(master, b"a" if choice == "abort" else b"\r")
|
||||
read_until(lambda: result.poll() is not None, timeout=45)
|
||||
expected_exit = 143 if choice == "daemon-term" else (0 if choice in {"skip", "early-skip", "retry"} else 130)
|
||||
assert result.returncode == expected_exit, output.decode(errors="replace")
|
||||
assert not psutil.pid_exists(first_hook.pid)
|
||||
_, live_processes = psutil.wait_procs(owned_processes, timeout=5)
|
||||
assert not [process.pid for process in live_processes if process.is_running() and process.status() != psutil.STATUS_ZOMBIE]
|
||||
assert b"Traceback" not in output, output.decode(errors="replace")
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl_after = Crawl.objects.get(id=crawl_id)
|
||||
snapshot_after = Snapshot.objects.get(id=snapshot["id"])
|
||||
if choice == "daemon-term":
|
||||
# A supervisor stop displaces this runner; the user's crawl
|
||||
# and snapshot remain resumable for the next owner.
|
||||
assert crawl_after.status == Crawl.StatusChoices.STARTED
|
||||
assert snapshot_after.status == Snapshot.StatusChoices.STARTED
|
||||
assert crawl_after.retry_at is not None and snapshot_after.retry_at is not None
|
||||
else:
|
||||
assert crawl_after.status == Crawl.StatusChoices.SEALED
|
||||
assert snapshot_after.status == Snapshot.StatusChoices.SEALED
|
||||
assert not Process.objects.filter(status="running", pid__isnull=False).exists()
|
||||
if choice in {"noninteractive", "daemon-tty", "daemon-noninteractive", "daemon-term"}:
|
||||
assert b"Choice [skip]:" not in output
|
||||
finally:
|
||||
if result is not None and result.poll() is None:
|
||||
result.terminate()
|
||||
try:
|
||||
result.wait(timeout=20)
|
||||
except subprocess.TimeoutExpired:
|
||||
result.send_signal(signal.SIGKILL)
|
||||
result.wait(timeout=5)
|
||||
for process in owned_processes:
|
||||
try:
|
||||
if process.is_running() and process.status() != psutil.STATUS_ZOMBIE:
|
||||
process.terminate()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
psutil.wait_procs(owned_processes, timeout=3)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,489 @@
|
||||
"""
|
||||
Tests for archivebox run CLI command.
|
||||
|
||||
Tests cover:
|
||||
- run with stdin JSONL (Crawl, Snapshot, ArchiveResult)
|
||||
- create-or-update behavior (records with/without id)
|
||||
- pass-through output (for chaining)
|
||||
"""
|
||||
|
||||
import os
|
||||
import pty
|
||||
import select
|
||||
import signal
|
||||
import subprocess
|
||||
import termios
|
||||
import time
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
from archivebox.tests.conftest import (
|
||||
cleanup_process_group,
|
||||
cli_env,
|
||||
create_test_url,
|
||||
parse_jsonl_output,
|
||||
pid_is_alive,
|
||||
run_archivebox_cmd,
|
||||
wait_for_log,
|
||||
wait_for_pid_to_disappear,
|
||||
)
|
||||
|
||||
from .test_cli_run_1 import (
|
||||
RUN_TEST_ENV as RUN_TEST_ENV,
|
||||
_install_real_chrome_for_test as _install_real_chrome_for_test,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.parametrize("choice", ["typed", "signal"])
|
||||
def test_run_third_interrupt_forces_chosen_abort(initialized_archive, choice):
|
||||
"""Direct run exits at once when Ctrl+C follows an explicit abort choice."""
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True", PLUGINS="chrome")
|
||||
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], cwd=initialized_archive, env=env)
|
||||
assert created.returncode == 0, created.stdout + created.stderr
|
||||
crawl_id = next(record["crawl_id"] for record in parse_jsonl_output(created.stdout) if record.get("type") == "Snapshot")
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 160))
|
||||
output = bytearray()
|
||||
result = None
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def active_hook_pid():
|
||||
with use_archivebox_db(initialized_archive):
|
||||
hook = Process.objects.filter(
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
|
||||
status="running",
|
||||
).first()
|
||||
return hook.pid if hook is not None and hook.is_running else None
|
||||
|
||||
try:
|
||||
result = run_archivebox_cmd(
|
||||
["run", f"--crawl-id={crawl_id}"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
read_until(lambda: active_hook_pid() is not None)
|
||||
hook_pid = active_hook_pid()
|
||||
assert hook_pid is not None
|
||||
owned = psutil.Process(result.pid).children(recursive=True)
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
if choice == "typed":
|
||||
os.write(master, b"a")
|
||||
else:
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Aborting crawl" in output, timeout=5)
|
||||
forced_at = time.monotonic()
|
||||
result.send_signal(signal.SIGINT)
|
||||
read_until(lambda: result.poll() is not None, timeout=5)
|
||||
assert time.monotonic() - forced_at < 2.0
|
||||
assert result.returncode == 130, output.decode(errors="replace")
|
||||
assert not pid_is_alive(hook_pid)
|
||||
assert not [child.pid for child in owned if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
|
||||
assert b"Traceback" not in output
|
||||
finally:
|
||||
if result is not None and result.poll() is None:
|
||||
result.terminate()
|
||||
result.wait(timeout=20)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_cli_run_retries_snapshot_after_filesystem_failure(initialized_archive):
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env(live=True, PLUGINS="hashes")
|
||||
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], cwd=initialized_archive, env=env)
|
||||
assert created.returncode == 0, created.stdout + created.stderr
|
||||
snapshot_id = next(record["id"] for record in parse_jsonl_output(created.stdout) if record.get("type") == "Snapshot")
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot = Snapshot.objects.get(pk=snapshot_id)
|
||||
crawl_id = str(snapshot.crawl_id)
|
||||
directory = snapshot.output_dir
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
(directory / "input.txt").write_text("Capture survives a temporary filesystem failure")
|
||||
broken_directory = directory / "hashes"
|
||||
broken_directory.symlink_to("hashes")
|
||||
|
||||
failed = run_archivebox_cmd(["run", f"--snapshot-id={snapshot_id}"], cwd=initialized_archive, env=env, timeout=60)
|
||||
assert failed.returncode == 1, failed.stdout + failed.stderr
|
||||
assert "FileExistsError: [Errno 17]" in failed.stderr
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot = Snapshot.objects.get(pk=snapshot_id)
|
||||
assert snapshot.status == Snapshot.StatusChoices.STARTED
|
||||
assert snapshot.retry_at is not None
|
||||
assert not ArchiveResult.objects.filter(snapshot=snapshot).exists()
|
||||
|
||||
broken_directory.unlink()
|
||||
resumed = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=initialized_archive, env=env, timeout=60)
|
||||
assert resumed.returncode == 0, resumed.stdout + resumed.stderr
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot = Snapshot.objects.get(pk=snapshot_id)
|
||||
assert snapshot.status == Snapshot.StatusChoices.SEALED
|
||||
result = snapshot.archiveresult_set.get(plugin="hashes")
|
||||
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
|
||||
assert result.output_size > 0
|
||||
assert (directory / "hashes").is_dir()
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.parametrize("precreate_snapshot", [False, True])
|
||||
def test_newest_crawl_runs_before_older_queued_and_started_crawls(initialized_archive, precreate_snapshot):
|
||||
from datetime import timedelta
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env(live=True, PLUGINS="hashes")
|
||||
old_url, queued_url, newest_url = create_test_url(), create_test_url(), create_test_url()
|
||||
old = run_archivebox_cmd(["snapshot", "create", old_url], cwd=initialized_archive, env=env)
|
||||
assert old.returncode == 0, old.stdout + old.stderr
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshot = Snapshot.objects.get(url=old_url)
|
||||
Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED)
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(retry_at=timezone.now() - timedelta(days=1))
|
||||
queued = run_archivebox_cmd(
|
||||
["snapshot" if precreate_snapshot else "crawl", "create", queued_url],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
assert queued.returncode == 0, queued.stdout + queued.stderr
|
||||
newest = run_archivebox_cmd(
|
||||
["snapshot" if precreate_snapshot else "crawl", "create", newest_url],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
)
|
||||
assert newest.returncode == 0, newest.stdout + newest.stderr
|
||||
|
||||
result = run_archivebox_cmd(["run", "--no-stdin"], cwd=initialized_archive, env=env, timeout=60)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert all(url in result.stdout for url in (old_url, queued_url, newest_url))
|
||||
assert result.stdout.index(newest_url) < result.stdout.index(queued_url) < result.stdout.index(old_url), result.stdout
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert (
|
||||
Snapshot.objects.filter(
|
||||
url__in=[old_url, queued_url, newest_url],
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
).count()
|
||||
== 3
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.timeout(660)
|
||||
def test_due_snapshots_share_one_crawl_scoped_chrome(initialized_archive, recursive_test_site):
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false")
|
||||
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
|
||||
urls = [recursive_test_site["root_url"], recursive_test_site["child_urls"][0]]
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.create(
|
||||
urls="\n".join(urls),
|
||||
config={"PLUGINS": "chrome", "CHROME_ISOLATION": "crawl"},
|
||||
created_by_id=get_or_create_system_user_pk(),
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now(),
|
||||
)
|
||||
for url in urls:
|
||||
Snapshot.objects.create(url=url, crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
|
||||
crawl_id = str(crawl.id)
|
||||
|
||||
result = run_archivebox_cmd(["run", "--no-stdin", f"--crawl-id={crawl_id}"], cwd=initialized_archive, env=env, timeout=600)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
with use_archivebox_db(initialized_archive):
|
||||
snapshots = Snapshot.objects.filter(crawl_id=crawl_id, url__in=urls, status=Snapshot.StatusChoices.SEALED)
|
||||
assert snapshots.count() == 2
|
||||
assert all((snapshot.output_dir / "chrome" / "navigation.json").is_file() for snapshot in snapshots)
|
||||
launches = Process.objects.filter(
|
||||
pwd__contains=crawl_id,
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
cmd__0__endswith="on_CrawlSetup__90_chrome_launch.daemon.bg.js",
|
||||
)
|
||||
assert launches.count() == 1, [(process.pid, process.exit_code) for process in launches]
|
||||
assert launches.get().exit_code == 0
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_busy_crawl_does_not_report_snapshot_progress(initialized_archive):
|
||||
from datetime import timedelta
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.base_models.models import get_or_create_system_user_pk
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.services.runner import _run_due_snapshot_id
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.create(
|
||||
urls="https://example.com/",
|
||||
config={"PLUGINS": "chrome", "CHROME_ISOLATION": "crawl"},
|
||||
created_by_id=get_or_create_system_user_pk(),
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now() + timedelta(minutes=1),
|
||||
)
|
||||
snapshot = Snapshot.objects.create(url=crawl.urls, crawl=crawl, retry_at=timezone.now())
|
||||
assert _run_due_snapshot_id(str(snapshot.id), lock_seconds=60, interactive_interrupts=False, runtime_config=None) is False
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.timeout(660)
|
||||
def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archive, recursive_test_site):
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false")
|
||||
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["snapshot", "create", recursive_test_site["root_url"]],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
snapshot_id = next(record["id"] for record in records if record.get("type") == "Snapshot")
|
||||
with use_archivebox_db(initialized_archive):
|
||||
browser_state = Snapshot.objects.get(id=snapshot_id).output_dir / "chrome" / "browser.json"
|
||||
|
||||
run_log = initialized_archive / "run-signal-chrome.log"
|
||||
run_log_handle = run_log.open("w", encoding="utf-8")
|
||||
run_process = run_archivebox_cmd(
|
||||
["run", f"--snapshot-id={snapshot_id}"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdout=run_log_handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
run_log_handle.close()
|
||||
try:
|
||||
wait_for_log(browser_state, '"ready": true', timeout=120)
|
||||
child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True) if pid_is_alive(child.pid)]
|
||||
assert child_pids
|
||||
|
||||
run_process.send_signal(signal.SIGTERM)
|
||||
run_process.wait(timeout=30)
|
||||
output = run_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert "Runner error" not in output
|
||||
for pid in child_pids:
|
||||
wait_for_pid_to_disappear(pid, timeout=15)
|
||||
finally:
|
||||
cleanup_process_group(run_process.pid)
|
||||
|
||||
|
||||
class TestRunWithArchiveResult:
|
||||
"""Tests for `archivebox run` with ArchiveResult input."""
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_run_treats_no_id_archiveresult_as_parent_snapshot_plugin_request(self, initialized_archive):
|
||||
import json
|
||||
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
create_result = run_archivebox_cmd(
|
||||
["snapshot", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
snapshot_id = next(record["id"] for record in parse_jsonl_output(create_result.stdout) if record.get("type") == "Snapshot")
|
||||
missing_hook = "on_Snapshot__99_missing_favicon_hook"
|
||||
request = json.dumps(
|
||||
{
|
||||
"type": "ArchiveResult",
|
||||
"snapshot_id": snapshot_id,
|
||||
"plugin": "favicon",
|
||||
"hook_name": missing_hook,
|
||||
"status": "queued",
|
||||
},
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=f"{request}\n",
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
with use_archivebox_db(initialized_archive):
|
||||
rows = list(
|
||||
ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="favicon").values_list(
|
||||
"hook_name",
|
||||
"status",
|
||||
"output_str",
|
||||
),
|
||||
)
|
||||
assert len(rows) == 1
|
||||
assert rows[0][0] != missing_hook
|
||||
assert rows[0][0].startswith("on_Snapshot__")
|
||||
assert rows[0][1] in ArchiveResult.FINAL_STATES
|
||||
|
||||
def test_run_requeues_failed_archiveresult(self, initialized_archive):
|
||||
"""Run uses a failed ArchiveResult as a parent Snapshot/plugin reference."""
|
||||
url = create_test_url()
|
||||
|
||||
# Create snapshot and archive result
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["snapshot", "create", url],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["archiveresult", "create", "--plugin=favicon"],
|
||||
stdin=stdout1,
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(stdout2))
|
||||
|
||||
initial_run = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=stdout2,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert initial_run.returncode == 0, initial_run.stderr
|
||||
persisted_result = run_archivebox_cmd(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert persisted_result.returncode == 0, persisted_result.stderr
|
||||
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(persisted_result.stdout))
|
||||
|
||||
# Update to failed
|
||||
update_result = run_archivebox_cmd(
|
||||
["archiveresult", "update", "--status=failed"],
|
||||
stdin=persisted_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert update_result.returncode == 0, update_result.stderr
|
||||
failed_result = run_archivebox_cmd(
|
||||
["archiveresult", "list", "--plugin=favicon"],
|
||||
cwd=initialized_archive,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert failed_result.returncode == 0, failed_result.stderr
|
||||
failed_records = [record for record in parse_jsonl_output(failed_result.stdout) if record.get("type") == "ArchiveResult"]
|
||||
assert len(failed_records) == 1
|
||||
assert failed_records[0]["status"] == "failed"
|
||||
failed_jsonl = next(line for line in failed_result.stdout.splitlines() if failed_records[0]["id"] in line) + "\n"
|
||||
|
||||
# Now run should re-queue it
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=failed_jsonl,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
stdout3, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
|
||||
assert code == 0
|
||||
records = parse_jsonl_output(stdout3)
|
||||
ar_records = [r for r in records if r.get("type") == "ArchiveResult"]
|
||||
assert len(ar_records) >= 1
|
||||
|
||||
|
||||
class TestRunMixedInput:
|
||||
"""Tests for `archivebox run` with mixed record types."""
|
||||
|
||||
def test_run_handles_mixed_records_emitted_by_cli(self, initialized_archive):
|
||||
"""Run handles real Crawl, Snapshot, and Tag records from CLI stages."""
|
||||
tag_result = run_archivebox_cmd(
|
||||
["tag", "create", "mixed-run-tag"],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert tag_result.returncode == 0, tag_result.stderr
|
||||
crawl_result = run_archivebox_cmd(
|
||||
["crawl", "create", create_test_url()],
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert crawl_result.returncode == 0, crawl_result.stderr
|
||||
snapshot_result = run_archivebox_cmd(
|
||||
["snapshot", "create"],
|
||||
stdin=crawl_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
assert snapshot_result.returncode == 0, snapshot_result.stderr
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["run"],
|
||||
stdin=tag_result.stdout + snapshot_result.stdout,
|
||||
cwd=initialized_archive,
|
||||
timeout=120,
|
||||
env=RUN_TEST_ENV,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
records = parse_jsonl_output(result.stdout)
|
||||
|
||||
types = {record.get("type") for record in records}
|
||||
assert {"Crawl", "Snapshot", "Tag"}.issubset(types)
|
||||
@@ -30,6 +30,7 @@ from archivebox.tests.conftest import (
|
||||
run_archivebox_cmd,
|
||||
start_archivebox_server,
|
||||
stop_archivebox_process,
|
||||
wait_for_log,
|
||||
wait_for_log_count,
|
||||
wait_for_pid_to_disappear,
|
||||
)
|
||||
@@ -588,7 +589,7 @@ def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_ru
|
||||
"from archivebox.crawls.models import Crawl;"
|
||||
"from archivebox.core.models import Snapshot;"
|
||||
"from archivebox.machine.models import Machine;"
|
||||
"machine = Machine.current(refresh=True);"
|
||||
"machine = Machine.current();"
|
||||
"machine.config = {**dict(machine.config or {}), 'SEARCH_BACKEND_ENGINE': 'sqlite'};"
|
||||
"machine.save(update_fields=['config', 'modified_at']);"
|
||||
"crawl = Crawl.objects.create("
|
||||
@@ -813,26 +814,32 @@ def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(initi
|
||||
try:
|
||||
server = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}.log", env=env)
|
||||
server_log = server.log_path
|
||||
# HTTP readiness can precede the foreground tail scope. Exercise its
|
||||
# actual signal policy, where interrupting Popen.poll stranded reaping.
|
||||
wait_for_log(server_log, "Tailing worker logs (Ctrl+C to stop)...")
|
||||
|
||||
os.kill(server.pid, stop_signal)
|
||||
server.wait(timeout=20)
|
||||
|
||||
if expected_notice:
|
||||
log_text = server_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert expected_notice in log_text
|
||||
assert "ArchiveBox server shut down gracefully" in log_text
|
||||
assert server.returncode == 0, log_text
|
||||
assert expected_notice in log_text, log_text
|
||||
assert "ArchiveBox server shut down gracefully" in log_text, log_text
|
||||
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
|
||||
|
||||
resumed = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}-resumed.log", env=env)
|
||||
resumed_log = resumed.log_path
|
||||
wait_for_log(resumed_log, "Tailing worker logs (Ctrl+C to stop)...")
|
||||
result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env, timeout=60)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
os.kill(resumed.pid, signal.SIGTERM)
|
||||
resumed.wait(timeout=20)
|
||||
resumed_text = resumed_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert "Got SIGTERM" in resumed_text
|
||||
assert "ArchiveBox server shut down gracefully" in resumed_text
|
||||
assert resumed.returncode == 0, resumed_text
|
||||
assert "Got SIGTERM" in resumed_text, resumed_text
|
||||
assert "ArchiveBox server shut down gracefully" in resumed_text, resumed_text
|
||||
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
|
||||
finally:
|
||||
for proc in (server, resumed):
|
||||
|
||||
@@ -194,6 +194,16 @@ class TestRequiredBinaryConfigHandling:
|
||||
|
||||
assert Path(resolved["WGET_BINARY"]).is_absolute()
|
||||
assert Path(resolved["WGET_BINARY"]).is_file()
|
||||
runtime_env = {**os.environ, **resolved}
|
||||
assert set(os.environ["PATH"].split(os.pathsep)).issubset(runtime_env["PATH"].split(os.pathsep))
|
||||
result = subprocess.run(
|
||||
["/usr/bin/env", "abxpkg", "--version"],
|
||||
env=runtime_env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_binary_env_var_name_only_handling(self, hermetic_lib_dir):
|
||||
|
||||
@@ -174,21 +174,32 @@ class TestMachineModel:
|
||||
|
||||
assert machine1.id == machine2.id
|
||||
|
||||
def test_machine_current_refreshes_after_interval(self):
|
||||
"""Machine.current() should refresh after recheck interval."""
|
||||
import archivebox.machine.models as models
|
||||
def test_machine_current_does_not_refresh_after_interval(self):
|
||||
"""A long-lived process must not turn a cached machine read into writes."""
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
machine1 = Machine.current()
|
||||
|
||||
# Manually expire the cache by modifying modified_at
|
||||
machine1.modified_at = timezone.now() - timedelta(seconds=MACHINE_RECHECK_INTERVAL + 1)
|
||||
machine1.save()
|
||||
models._CURRENT_MACHINE = machine1
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
machine2 = Machine.current()
|
||||
assert Machine.current() is machine1
|
||||
assert machine2 is machine1
|
||||
assert list(queries) == []
|
||||
|
||||
machine2 = Machine.current()
|
||||
def test_machine_config_save_updates_cache_without_reinitializing(self):
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
# Should have fetched/updated the machine (same GUID)
|
||||
assert machine1.guid == machine2.guid
|
||||
cached = Machine.current()
|
||||
edited = Machine.objects.get(pk=cached.pk)
|
||||
edited.config = {**edited.config, "USER_AGENT": "read-only-machine-test"}
|
||||
edited.save(update_fields=["config"])
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
current = Machine.current()
|
||||
assert current is cached
|
||||
assert current.config["USER_AGENT"] == "read-only-machine-test"
|
||||
assert list(queries) == []
|
||||
|
||||
def test_machine_current_recreates_stale_cached_row(self):
|
||||
"""Machine.current() should recreate the cached machine if the row was deleted."""
|
||||
@@ -284,9 +295,9 @@ class TestMachineModel:
|
||||
"WGET_BINARY": "/tmp/archivebox-test-missing-wget",
|
||||
}
|
||||
machine.save(update_fields=["config"])
|
||||
models._CURRENT_MACHINE = machine
|
||||
models._CURRENT_MACHINE = None
|
||||
|
||||
refreshed = Machine.current(refresh=True)
|
||||
refreshed = Machine.current()
|
||||
|
||||
# Valid binary paths inside ABXPKG_LIB_DIR survive.
|
||||
assert refreshed.config.get("CHROME_BINARY") == str(chrome_path)
|
||||
|
||||
@@ -189,36 +189,28 @@ def test_migration_preserves_supported_extended_08_metadata(migration_08_data, u
|
||||
assert migrated_counts == expected_counts
|
||||
|
||||
|
||||
def test_migration_preserves_snapshot_count(migration_08_data):
|
||||
"""Migration should preserve all snapshots from 0.8.x."""
|
||||
def test_migration_preserves_08_collection(migration_08_data):
|
||||
"""One real upgrade preserves every field and relationship in the same collection."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
expected_count = len(original_data["snapshots"])
|
||||
with sqlite3.connect(db_path) as conn:
|
||||
original_tag_count = conn.execute("SELECT COUNT(*) FROM core_snapshot_tags").fetchone()[0]
|
||||
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
# migration_preserves_snapshot_count
|
||||
expected_count = len(original_data["snapshots"])
|
||||
|
||||
ok, msg = verify_snapshot_count(db_path, expected_count)
|
||||
assert ok, msg
|
||||
|
||||
|
||||
def test_migration_preserves_snapshot_urls(migration_08_data):
|
||||
"""Migration should preserve all snapshot URLs from 0.8.x."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
# migration_preserves_snapshot_urls
|
||||
expected_urls = [s["url"] for s in original_data["snapshots"]]
|
||||
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
ok, msg = verify_snapshot_urls(db_path, expected_urls)
|
||||
assert ok, msg
|
||||
|
||||
|
||||
def test_migration_preserves_crawls(migration_08_data):
|
||||
"""Migration should preserve all Crawl records and create default crawl if needed."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
# migration_preserves_crawls
|
||||
# Count snapshots with NULL crawl_id in original data
|
||||
snapshots_without_crawl = sum(1 for s in original_data["snapshots"] if s["crawl_id"] is None)
|
||||
|
||||
@@ -230,6 +222,191 @@ def test_migration_preserves_crawls(migration_08_data):
|
||||
ok, msg = verify_crawl_count(db_path, expected_count)
|
||||
assert ok, msg
|
||||
|
||||
# migration_preserves_snapshot_crawl_links
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check EVERY snapshot has a crawl_id after migration
|
||||
for snapshot in original_data["snapshots"]:
|
||||
cursor.execute("SELECT crawl_id FROM core_snapshot WHERE url = ?", (snapshot["url"],))
|
||||
row = cursor.fetchone()
|
||||
assert row is not None, f"Snapshot {snapshot['url']} not found after migration"
|
||||
|
||||
if snapshot["crawl_id"] is not None:
|
||||
# Snapshots that had a crawl should keep it
|
||||
assert row[0] == snapshot["crawl_id"], f"Crawl ID changed for {snapshot['url']}: expected {snapshot['crawl_id']}, got {row[0]}"
|
||||
else:
|
||||
# Snapshots without a crawl should now have one (the default crawl)
|
||||
assert row[0] is not None, f"Snapshot {snapshot['url']} should have been assigned to default crawl but has NULL"
|
||||
|
||||
conn.close()
|
||||
|
||||
# migration_preserves_tags
|
||||
ok, msg = verify_tag_count(db_path, len(original_data["tags"]))
|
||||
assert ok, msg
|
||||
|
||||
# migration_preserves_archiveresults
|
||||
expected_count = len(original_data["archiveresults"])
|
||||
expected_counts = {}
|
||||
for result in original_data["archiveresults"]:
|
||||
status = "succeeded" if result["status"] == "success" else result["status"]
|
||||
key = (result["extractor"], status)
|
||||
expected_counts[key] = expected_counts.get(key, 0) + 1
|
||||
|
||||
ok, msg = verify_archiveresult_count(db_path, expected_count)
|
||||
assert ok, msg
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status")
|
||||
migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()}
|
||||
cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL")
|
||||
missing_process_count = cursor.fetchone()[0]
|
||||
cursor.execute("SELECT COUNT(*) FROM machine_process")
|
||||
process_count = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
assert migrated_counts == expected_counts
|
||||
assert missing_process_count == 0
|
||||
assert process_count == expected_count
|
||||
|
||||
# migration_preserves_archiveresult_status
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get status counts
|
||||
cursor.execute("SELECT status, COUNT(*) FROM core_archiveresult GROUP BY status")
|
||||
status_counts = dict(cursor.fetchall())
|
||||
conn.close()
|
||||
|
||||
# Original data has known status distribution: succeeded, failed, skipped
|
||||
assert "succeeded" in status_counts, "Should have succeeded results"
|
||||
assert "failed" in status_counts, "Should have failed results"
|
||||
assert "skipped" in status_counts, "Should have skipped results"
|
||||
|
||||
# migration_preserves_snapshot_titles
|
||||
expected_titles = {s["url"]: s["title"] for s in original_data["snapshots"]}
|
||||
|
||||
ok, msg = verify_snapshot_titles(db_path, expected_titles)
|
||||
assert ok, msg
|
||||
|
||||
# migration_preserves_foreign_keys
|
||||
ok, msg = verify_foreign_keys(db_path)
|
||||
assert ok, msg
|
||||
|
||||
# migration_creates_process_records
|
||||
# Verify Process records created
|
||||
expected_count = len(original_data["archiveresults"])
|
||||
ok, msg = verify_process_migration(db_path, expected_count)
|
||||
assert ok, msg
|
||||
|
||||
# migration_creates_binary_records
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check Binary records exist
|
||||
cursor.execute("SELECT COUNT(*) FROM machine_binary")
|
||||
binary_count = cursor.fetchone()[0]
|
||||
|
||||
# Should have at least one binary per unique extractor
|
||||
extractors = {ar["extractor"] for ar in original_data["archiveresults"]}
|
||||
assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}"
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*)
|
||||
FROM machine_process
|
||||
WHERE cmd != '[]' AND binary_id IS NULL
|
||||
""")
|
||||
missing_binary_count = cursor.fetchone()[0]
|
||||
assert missing_binary_count == 0
|
||||
|
||||
cursor.execute("""
|
||||
SELECT p.cmd, b.name, b.abspath
|
||||
FROM machine_process p
|
||||
JOIN machine_binary b ON p.binary_id = b.id
|
||||
WHERE p.cmd != '[]'
|
||||
""")
|
||||
rows = cursor.fetchall()
|
||||
assert rows
|
||||
for cmd_raw, binary_name, binary_abspath in rows:
|
||||
cmd = json.loads(cmd_raw)
|
||||
assert binary_name == cmd[0]
|
||||
assert binary_abspath == cmd[0]
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM machine_process WHERE iface_id IS NULL")
|
||||
missing_iface_count = cursor.fetchone()[0]
|
||||
assert missing_iface_count == 0
|
||||
|
||||
conn.close()
|
||||
|
||||
# migration_preserves_cmd_data
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check that Process records have cmd arrays
|
||||
cursor.execute("SELECT cmd FROM machine_process WHERE cmd != '[]'")
|
||||
cmd_records = cursor.fetchall()
|
||||
|
||||
# All Processes should have non-empty cmd (test data has json.dumps([extractor, '--version']))
|
||||
expected_count = len(original_data["archiveresults"])
|
||||
assert len(cmd_records) == expected_count, f"Expected {expected_count} Processes with cmd, got {len(cmd_records)}"
|
||||
|
||||
conn.close()
|
||||
|
||||
# no_duplicate_snapshots_after_migration
|
||||
# Check for duplicate URLs
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT url, COUNT(*) as cnt FROM core_snapshot
|
||||
GROUP BY url HAVING cnt > 1
|
||||
""")
|
||||
duplicates = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
assert len(duplicates) == 0, f"Found duplicate URLs: {duplicates}"
|
||||
|
||||
# no_orphaned_archiveresults_after_migration
|
||||
ok, msg = verify_foreign_keys(db_path)
|
||||
assert ok, msg
|
||||
|
||||
# timestamps_preserved_after_migration
|
||||
original_timestamps = {s["url"]: s["timestamp"] for s in original_data["snapshots"]}
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT url, timestamp FROM core_snapshot")
|
||||
migrated_timestamps = {row[0]: row[1] for row in cursor.fetchall()}
|
||||
conn.close()
|
||||
|
||||
for url, original_ts in original_timestamps.items():
|
||||
assert migrated_timestamps.get(url) == original_ts, f"Timestamp changed for {url}: {original_ts} -> {migrated_timestamps.get(url)}"
|
||||
|
||||
# crawl_data_preserved_after_migration
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check each crawl's data is preserved
|
||||
for crawl in original_data["crawls"]:
|
||||
cursor.execute("SELECT urls, label, status FROM crawls_crawl WHERE id = ?", (crawl["id"],))
|
||||
row = cursor.fetchone()
|
||||
assert row is not None, f"Crawl {crawl['id']} not found after migration"
|
||||
assert row[0] == crawl["urls"], f"URLs mismatch for crawl {crawl['id']}"
|
||||
assert row[1] == crawl["label"], f"Label mismatch for crawl {crawl['id']}"
|
||||
assert row[2] == crawl["status"], f"Status mismatch for crawl {crawl['id']}"
|
||||
|
||||
conn.close()
|
||||
|
||||
# tag_associations_preserved_after_migration
|
||||
# Count tag associations after migration
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
|
||||
migrated_count = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
assert migrated_count == original_tag_count, f"Tag associations changed: {original_tag_count} -> {migrated_count}"
|
||||
|
||||
|
||||
def test_legacy_default_crawl_is_sealed_without_a_due_retry(tmp_path):
|
||||
"""Upgrading a legacy collection must not leave its sealed crawl runnable."""
|
||||
@@ -343,72 +520,6 @@ def test_forward_migration_normalizes_active_legacy_retry_timestamps(migration_0
|
||||
assert conn.execute("SELECT retry_at FROM machine_process WHERE id = ?", (process_id,)).fetchone() == (expected_utc_timestamp,)
|
||||
|
||||
|
||||
def test_migration_preserves_snapshot_crawl_links(migration_08_data):
|
||||
"""Migration should preserve snapshot-to-crawl relationships and assign default crawl to orphans."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check EVERY snapshot has a crawl_id after migration
|
||||
for snapshot in original_data["snapshots"]:
|
||||
cursor.execute("SELECT crawl_id FROM core_snapshot WHERE url = ?", (snapshot["url"],))
|
||||
row = cursor.fetchone()
|
||||
assert row is not None, f"Snapshot {snapshot['url']} not found after migration"
|
||||
|
||||
if snapshot["crawl_id"] is not None:
|
||||
# Snapshots that had a crawl should keep it
|
||||
assert row[0] == snapshot["crawl_id"], f"Crawl ID changed for {snapshot['url']}: expected {snapshot['crawl_id']}, got {row[0]}"
|
||||
else:
|
||||
# Snapshots without a crawl should now have one (the default crawl)
|
||||
assert row[0] is not None, f"Snapshot {snapshot['url']} should have been assigned to default crawl but has NULL"
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_migration_preserves_tags(migration_08_data):
|
||||
"""Migration should preserve all tags."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
ok, msg = verify_tag_count(db_path, len(original_data["tags"]))
|
||||
assert ok, msg
|
||||
|
||||
|
||||
def test_migration_preserves_archiveresults(migration_08_data):
|
||||
"""Migration should preserve ArchiveResult rows and link each one to a Process."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
expected_count = len(original_data["archiveresults"])
|
||||
expected_counts = {}
|
||||
for result in original_data["archiveresults"]:
|
||||
status = "succeeded" if result["status"] == "success" else result["status"]
|
||||
key = (result["extractor"], status)
|
||||
expected_counts[key] = expected_counts.get(key, 0) + 1
|
||||
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
ok, msg = verify_archiveresult_count(db_path, expected_count)
|
||||
assert ok, msg
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status")
|
||||
migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()}
|
||||
cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL")
|
||||
missing_process_count = cursor.fetchone()[0]
|
||||
cursor.execute("SELECT COUNT(*) FROM machine_process")
|
||||
process_count = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
assert migrated_counts == expected_counts
|
||||
assert missing_process_count == 0
|
||||
assert process_count == expected_count
|
||||
|
||||
|
||||
def test_migration_preserves_users_groups_permissions_api_secrets_and_repeated_init(migration_08_data):
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
|
||||
@@ -446,26 +557,6 @@ def test_process_metadata_migration_rolls_back_completely_on_malformed_row(migra
|
||||
assert conn.execute("SELECT COUNT(*) FROM machine_binary").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_migration_preserves_archiveresult_status(migration_08_data):
|
||||
"""Migration should preserve archive result status values."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get status counts
|
||||
cursor.execute("SELECT status, COUNT(*) FROM core_archiveresult GROUP BY status")
|
||||
status_counts = dict(cursor.fetchall())
|
||||
conn.close()
|
||||
|
||||
# Original data has known status distribution: succeeded, failed, skipped
|
||||
assert "succeeded" in status_counts, "Should have succeeded results"
|
||||
assert "failed" in status_counts, "Should have failed results"
|
||||
assert "skipped" in status_counts, "Should have skipped results"
|
||||
|
||||
|
||||
def test_status_works_after_migration(migration_08_data):
|
||||
"""Status command should work after migration."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
@@ -506,28 +597,6 @@ def test_search_works_after_migration(migration_08_data):
|
||||
assert ok, msg
|
||||
|
||||
|
||||
def test_migration_preserves_snapshot_titles(migration_08_data):
|
||||
"""Migration should preserve all snapshot titles."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
expected_titles = {s["url"]: s["title"] for s in original_data["snapshots"]}
|
||||
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
ok, msg = verify_snapshot_titles(db_path, expected_titles)
|
||||
assert ok, msg
|
||||
|
||||
|
||||
def test_migration_preserves_foreign_keys(migration_08_data):
|
||||
"""Migration should maintain foreign key relationships."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
ok, msg = verify_foreign_keys(db_path)
|
||||
assert ok, msg
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_snapshot_status", (True, False), ids=("with-status", "without-status"))
|
||||
def test_migration_preserves_08_timestamp_meanings(migration_08_data, has_snapshot_status):
|
||||
"""0.8.x already has separated timestamp/bookmarked_at/created_at/downloaded_at fields."""
|
||||
@@ -652,174 +721,6 @@ def test_version_works_after_migration(migration_08_data):
|
||||
assert "ArchiveBox" in output or "version" in output.lower(), f"Version output missing expected content: {output[:500]}"
|
||||
|
||||
|
||||
def test_migration_creates_process_records(migration_08_data):
|
||||
"""Migration should create Process records for all ArchiveResults."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
# Verify Process records created
|
||||
expected_count = len(original_data["archiveresults"])
|
||||
ok, msg = verify_process_migration(db_path, expected_count)
|
||||
assert ok, msg
|
||||
|
||||
|
||||
def test_migration_creates_binary_records(migration_08_data):
|
||||
"""Migration should create and link Binary/NetworkInterface records from migrated Process data."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check Binary records exist
|
||||
cursor.execute("SELECT COUNT(*) FROM machine_binary")
|
||||
binary_count = cursor.fetchone()[0]
|
||||
|
||||
# Should have at least one binary per unique extractor
|
||||
extractors = {ar["extractor"] for ar in original_data["archiveresults"]}
|
||||
assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}"
|
||||
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*)
|
||||
FROM machine_process
|
||||
WHERE cmd != '[]' AND binary_id IS NULL
|
||||
""")
|
||||
missing_binary_count = cursor.fetchone()[0]
|
||||
assert missing_binary_count == 0
|
||||
|
||||
cursor.execute("""
|
||||
SELECT p.cmd, b.name, b.abspath
|
||||
FROM machine_process p
|
||||
JOIN machine_binary b ON p.binary_id = b.id
|
||||
WHERE p.cmd != '[]'
|
||||
""")
|
||||
rows = cursor.fetchall()
|
||||
assert rows
|
||||
for cmd_raw, binary_name, binary_abspath in rows:
|
||||
cmd = json.loads(cmd_raw)
|
||||
assert binary_name == cmd[0]
|
||||
assert binary_abspath == cmd[0]
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM machine_process WHERE iface_id IS NULL")
|
||||
missing_iface_count = cursor.fetchone()[0]
|
||||
assert missing_iface_count == 0
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_migration_preserves_cmd_data(migration_08_data):
|
||||
"""Migration should preserve cmd data in Process.cmd field."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check that Process records have cmd arrays
|
||||
cursor.execute("SELECT cmd FROM machine_process WHERE cmd != '[]'")
|
||||
cmd_records = cursor.fetchall()
|
||||
|
||||
# All Processes should have non-empty cmd (test data has json.dumps([extractor, '--version']))
|
||||
expected_count = len(original_data["archiveresults"])
|
||||
assert len(cmd_records) == expected_count, f"Expected {expected_count} Processes with cmd, got {len(cmd_records)}"
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_no_duplicate_snapshots_after_migration(migration_08_data):
|
||||
"""Migration should not create duplicate snapshots."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
# Check for duplicate URLs
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
SELECT url, COUNT(*) as cnt FROM core_snapshot
|
||||
GROUP BY url HAVING cnt > 1
|
||||
""")
|
||||
duplicates = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
assert len(duplicates) == 0, f"Found duplicate URLs: {duplicates}"
|
||||
|
||||
|
||||
def test_no_orphaned_archiveresults_after_migration(migration_08_data):
|
||||
"""Migration should not leave orphaned ArchiveResults."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
ok, msg = verify_foreign_keys(db_path)
|
||||
assert ok, msg
|
||||
|
||||
|
||||
def test_timestamps_preserved_after_migration(migration_08_data):
|
||||
"""Migration should preserve original timestamps."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
original_timestamps = {s["url"]: s["timestamp"] for s in original_data["snapshots"]}
|
||||
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT url, timestamp FROM core_snapshot")
|
||||
migrated_timestamps = {row[0]: row[1] for row in cursor.fetchall()}
|
||||
conn.close()
|
||||
|
||||
for url, original_ts in original_timestamps.items():
|
||||
assert migrated_timestamps.get(url) == original_ts, f"Timestamp changed for {url}: {original_ts} -> {migrated_timestamps.get(url)}"
|
||||
|
||||
|
||||
def test_crawl_data_preserved_after_migration(migration_08_data):
|
||||
"""Migration should preserve crawl metadata (urls, label, status)."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check each crawl's data is preserved
|
||||
for crawl in original_data["crawls"]:
|
||||
cursor.execute("SELECT urls, label, status FROM crawls_crawl WHERE id = ?", (crawl["id"],))
|
||||
row = cursor.fetchone()
|
||||
assert row is not None, f"Crawl {crawl['id']} not found after migration"
|
||||
assert row[0] == crawl["urls"], f"URLs mismatch for crawl {crawl['id']}"
|
||||
assert row[1] == crawl["label"], f"Label mismatch for crawl {crawl['id']}"
|
||||
assert row[2] == crawl["status"], f"Status mismatch for crawl {crawl['id']}"
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_tag_associations_preserved_after_migration(migration_08_data):
|
||||
"""Migration should preserve snapshot-tag associations."""
|
||||
work_dir, db_path, original_data = migration_08_data
|
||||
# Count tag associations before migration
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
|
||||
original_count = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
|
||||
assert result.returncode == 0, f"Init failed: {result.stderr}"
|
||||
|
||||
# Count tag associations after migration
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
|
||||
migrated_count = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
|
||||
assert migrated_count == original_count, f"Tag associations changed: {original_count} -> {migrated_count}"
|
||||
|
||||
|
||||
def test_update_migrates_db_snapshot_when_legacy_index_missing(tmp_path):
|
||||
"""A legacy folder with no index file should still migrate if its timestamp exists in DB."""
|
||||
work_dir = tmp_path
|
||||
|
||||
@@ -1,695 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FutureTimeoutError
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qs, quote, urlsplit
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from asgiref.testing import ApplicationCommunicator
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
def _reset_runtime_config() -> None:
|
||||
from archivebox.config import common
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
for value in vars(common).values():
|
||||
cache_clear = getattr(value, "cache_clear", None)
|
||||
if cache_clear is not None:
|
||||
cache_clear()
|
||||
Machine.current(refresh=True)
|
||||
|
||||
|
||||
def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
|
||||
os.chdir(data_dir)
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", *values],
|
||||
cwd=data_dir,
|
||||
env=env,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
_reset_runtime_config()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opencode_archive_config(initialized_archive):
|
||||
port = _free_port()
|
||||
state_dir = initialized_archive / "opencode"
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
|
||||
"OPENCODE_ENABLED": "True",
|
||||
"OPENCODE_HOST": "127.0.0.1",
|
||||
"OPENCODE_PORT": str(port),
|
||||
"OPENCODE_WORKDIR": str(initialized_archive),
|
||||
"OPENCODE_STATE_DIR": str(state_dir),
|
||||
"OPENCODE_TIMEOUT": "60",
|
||||
},
|
||||
)
|
||||
_set_archivebox_config(
|
||||
initialized_archive,
|
||||
"OPENCODE_ENABLED=True",
|
||||
"OPENCODE_HOST=127.0.0.1",
|
||||
f"OPENCODE_PORT={port}",
|
||||
f"OPENCODE_WORKDIR={initialized_archive}",
|
||||
f"OPENCODE_STATE_DIR={state_dir}",
|
||||
"OPENCODE_TIMEOUT=60",
|
||||
env=env,
|
||||
)
|
||||
return SimpleNamespace(data_dir=initialized_archive, port=port, state_dir=state_dir, env=env)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def installed_opencode(opencode_archive_config):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
install = run_archivebox_cmd(
|
||||
["install", "opencode", "--binproviders=env,pnpm"],
|
||||
cwd=opencode_archive_config.data_dir,
|
||||
env=opencode_archive_config.env,
|
||||
timeout=1200,
|
||||
)
|
||||
assert install.returncode == 0, install.stderr or install.stdout
|
||||
_reset_runtime_config()
|
||||
|
||||
config = get_config().model_dump(mode="json")
|
||||
settings = runtime._settings(config, opencode_archive_config.data_dir)
|
||||
settings["archivebox_base_url"] = "http://admin.archivebox.localhost:5797"
|
||||
settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:5797/admin"
|
||||
settings["archivebox_api_url"] = "http://admin.archivebox.localhost:5797/api/"
|
||||
binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"])
|
||||
version = binary.exec(
|
||||
cmd=("--version",),
|
||||
env={**os.environ, **binary_env},
|
||||
timeout=120,
|
||||
)
|
||||
assert version.returncode == 0, version.stderr or version.stdout
|
||||
return SimpleNamespace(config=opencode_archive_config, settings=settings)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_opencode(installed_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
settings = installed_opencode.settings
|
||||
ok, error = runtime._ensure_opencode(settings)
|
||||
assert ok, error
|
||||
|
||||
process = runtime._PROCESS
|
||||
assert process is not None
|
||||
try:
|
||||
yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process)
|
||||
finally:
|
||||
runtime._stop_owned_process()
|
||||
|
||||
|
||||
def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
os.chdir(initialized_archive)
|
||||
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
|
||||
_reset_runtime_config()
|
||||
assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False
|
||||
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None
|
||||
|
||||
|
||||
def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive):
|
||||
_set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False")
|
||||
|
||||
assert get_config().OPENCODE_ENABLED is False
|
||||
assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404
|
||||
for path in ("/add/", "/admin/core/snapshot/"):
|
||||
response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200
|
||||
assert b'href="/admin/agent"' not in response.content
|
||||
|
||||
|
||||
def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode):
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 302
|
||||
assert "/admin/login/" in response.headers["Location"]
|
||||
|
||||
next_path = "/admin/agent?x=1&next=https://example.com"
|
||||
response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 302
|
||||
assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]}
|
||||
|
||||
user = django_user_model.objects.create_user(username="regular", password="testpassword")
|
||||
client.force_login(user)
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_proxy_blocks_cross_origin_mutation(admin_client, db, live_opencode):
|
||||
response = admin_client.post(
|
||||
"/admin/agent/opencode/session",
|
||||
data=b"{}",
|
||||
content_type="application/json",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_ORIGIN="https://evil.example",
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_opencode):
|
||||
response = admin_client.post(
|
||||
"/admin/agent/opencode/session",
|
||||
data=b"{}",
|
||||
content_type="application/json",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="cross-site",
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_cold_agent_wrapper_returns_before_server_starts(admin_client, installed_opencode):
|
||||
import time
|
||||
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
assert not runtime._owned_process_running()
|
||||
started = time.monotonic()
|
||||
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert time.monotonic() - started < 3
|
||||
assert not runtime._owned_process_running()
|
||||
assert b'id="opencode-agent-welcome"' in response.content
|
||||
assert f'<iframe src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
|
||||
|
||||
|
||||
def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
frame_path = runtime._project_route(live_opencode.config.data_dir)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert f'<iframe src="{frame_path}"'.encode() in response.content
|
||||
assert b'id="opencode-agent-welcome"' in response.content
|
||||
assert b'id="header"' in response.content
|
||||
assert b'id="progress-monitor"' in response.content
|
||||
assert b'<a href="/admin/agent" class="navbar-item navbar-ai">' in response.content
|
||||
add_page = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert add_page.status_code == 200
|
||||
assert '<a href="/admin/agent">💬 Crawl with AI</a>'.encode() in add_page.content
|
||||
assert response.context["proxy_prefix"] == runtime._PROXY_PREFIX
|
||||
assert b"/_archivebox/health" not in response.content
|
||||
assert b"window.setInterval(check, 3000)" not in response.content
|
||||
assert response.headers["X-Frame-Options"] == "DENY"
|
||||
assert response.headers["Content-Security-Policy"] == "frame-ancestors 'none'"
|
||||
|
||||
frame = admin_client.get(
|
||||
frame_path,
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert frame.status_code == 302
|
||||
assert frame.headers["Location"].startswith(frame_path + "/")
|
||||
session = admin_client.get(
|
||||
frame.headers["Location"],
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert session.status_code == 200
|
||||
assert session.headers["X-Frame-Options"] == "SAMEORIGIN"
|
||||
assert session.headers["Content-Security-Policy"] == "frame-ancestors 'self'"
|
||||
|
||||
|
||||
def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):
|
||||
workdir = str(live_opencode.config.data_dir.resolve())
|
||||
encoded_workdir = quote(workdir)
|
||||
|
||||
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert agent.status_code == 200
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
frame_path = runtime._project_route(live_opencode.config.data_dir)
|
||||
frame = admin_client.get(frame_path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert frame.status_code == 302
|
||||
session_id = frame.headers["Location"].rsplit("/", 1)[-1]
|
||||
|
||||
project = admin_client.get(
|
||||
f"/admin/agent/opencode/project/current?directory={encoded_workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert project.status_code == 200
|
||||
assert project.json()["id"] == "global"
|
||||
assert not project.json().get("vcs")
|
||||
|
||||
path = admin_client.get(
|
||||
f"/admin/agent/opencode/path?directory={encoded_workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert path.status_code == 200
|
||||
assert path.json()["directory"] == workdir
|
||||
|
||||
sessions = admin_client.get(
|
||||
f"/admin/agent/opencode/session?directory={encoded_workdir}&roots=true&limit=55",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert sessions.status_code == 200
|
||||
assert any(session["id"] == session_id and session["directory"] == workdir for session in sessions.json())
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
|
||||
|
||||
def test_opencode_proxy_restarts_server_for_an_existing_agent_page(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
old_process = runtime._PROCESS
|
||||
runtime._stop_owned_process()
|
||||
|
||||
response = admin_client.get(
|
||||
"/admin/agent/opencode/global/health",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert runtime._PROCESS is not None
|
||||
assert runtime._PROCESS is not old_process
|
||||
assert runtime._PROCESS.poll() is None
|
||||
|
||||
|
||||
def test_concurrent_opencode_startup_waits_until_server_is_ready(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
runtime._stop_owned_process()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(runtime._ensure_opencode, [live_opencode.settings] * 2))
|
||||
|
||||
assert results == [(True, ""), (True, "")]
|
||||
assert runtime._health(live_opencode.settings)
|
||||
|
||||
|
||||
def test_opencode_does_not_probe_or_replace_a_ready_owned_process(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
process = runtime._PROCESS
|
||||
settings = {**live_opencode.settings, "port": _free_port()}
|
||||
settings["origin"] = f"http://{settings['host']}:{settings['port']}"
|
||||
|
||||
ok, error = runtime._ensure_opencode(settings)
|
||||
|
||||
assert ok, error
|
||||
assert process is not None
|
||||
assert runtime._PROCESS is process
|
||||
assert process.poll() is None
|
||||
|
||||
|
||||
def test_opencode_proxy_does_not_wait_for_recovery_lock(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
workdir = quote(str(live_opencode.config.data_dir.resolve()))
|
||||
assert runtime._owned_process_ready()
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
runtime._PROCESS_LOCK.acquire()
|
||||
try:
|
||||
request = executor.submit(
|
||||
admin_client.get,
|
||||
f"/admin/agent/opencode/path?directory={workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
response = request.result(timeout=5)
|
||||
finally:
|
||||
runtime._PROCESS_LOCK.release()
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert str(live_opencode.config.data_dir.resolve()).encode() in response.content
|
||||
|
||||
|
||||
def test_opencode_proxy_waits_for_owned_process_readiness(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
process = runtime._PROCESS
|
||||
assert process is not None
|
||||
runtime._PROCESS_READY = None
|
||||
workdir = quote(str(live_opencode.config.data_dir.resolve()))
|
||||
|
||||
response = admin_client.get(
|
||||
f"/admin/agent/opencode/path?directory={workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert runtime._PROCESS is process
|
||||
assert runtime._PROCESS_READY is process
|
||||
|
||||
|
||||
def test_opencode_proxy_preserves_protocol_headers(admin_client, live_opencode):
|
||||
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
|
||||
created = admin_client.post(
|
||||
"/admin/agent/opencode/pty",
|
||||
data={"command": "/bin/sh", "args": []},
|
||||
content_type="application/json",
|
||||
**headers,
|
||||
)
|
||||
assert created.status_code == 200
|
||||
path = f"/admin/agent/opencode/pty/{created.json()['id']}"
|
||||
try:
|
||||
response = admin_client.post(path + "/connect-token", HTTP_X_OPENCODE_TICKET="1", **headers)
|
||||
assert response.status_code == 200, response.content
|
||||
assert response.json()["ticket"]
|
||||
finally:
|
||||
deleted = admin_client.delete(path, **headers)
|
||||
assert deleted.status_code == 200, deleted.content
|
||||
|
||||
|
||||
def test_opencode_oauth_callback_waits_for_user_and_preserves_cancellation(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
settings = {**live_opencode.settings, "timeout": 1}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
status, _, body = runtime.proxy(settings, "POST", "provider/openai/oauth/authorize", (), headers, b'{"method":0}')
|
||||
assert status == 200
|
||||
authorization = json.loads(body)
|
||||
assert urlsplit(authorization["url"]).hostname == "auth.openai.com"
|
||||
redirect = parse_qs(urlsplit(authorization["url"]).query)["redirect_uri"][0]
|
||||
callback_origin = urlsplit(redirect)
|
||||
assert callback_origin.hostname == "localhost"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
callback = executor.submit(runtime.proxy, settings, "POST", "provider/openai/oauth/callback", (), headers, b'{"method":0}')
|
||||
try:
|
||||
# A real pending authorization must outlive the ordinary API read
|
||||
# timeout. No tokens or substituted provider responses are used.
|
||||
with pytest.raises(FutureTimeoutError):
|
||||
callback.result(timeout=2)
|
||||
finally:
|
||||
cancelled = requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=5)
|
||||
assert cancelled.status_code == 200
|
||||
assert cancelled.text == "Login cancelled"
|
||||
status, response_headers, body = callback.result(timeout=5)
|
||||
assert status == 500
|
||||
assert response_headers["Content-Type"].startswith("application/json")
|
||||
error = json.loads(body)
|
||||
assert error["name"] == "UnknownError"
|
||||
assert error["data"]["ref"].startswith("err_")
|
||||
runtime._stop_owned_process()
|
||||
# The pnpm launcher can exit before its server child. Stopping the owned
|
||||
# process must release both listeners so another collection can authorize.
|
||||
with pytest.raises(requests.ConnectionError):
|
||||
requests.get(settings["origin"] + "/global/health", timeout=2)
|
||||
with pytest.raises(requests.ConnectionError):
|
||||
requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=2)
|
||||
|
||||
|
||||
def test_opencode_static_assets_cache_privately_and_revalidate(admin_client, live_opencode):
|
||||
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
|
||||
page = admin_client.get("/admin/agent/opencode/", **headers)
|
||||
assert page.status_code == 200
|
||||
asset = re.search(rb'src="(/admin/agent/opencode/assets/[^\"]+\.js)"', page.content)
|
||||
assert asset is not None
|
||||
path = asset[1].decode()
|
||||
response = admin_client.get(path, **headers)
|
||||
assert response.status_code == 200
|
||||
assert "private" in response.headers["Cache-Control"]
|
||||
assert "max-age=" in response.headers["Cache-Control"]
|
||||
assert b"/admin/agent/opencode" in response.content
|
||||
repeated = admin_client.get(path, HTTP_IF_NONE_MATCH=response.headers["ETag"], **headers)
|
||||
assert repeated.status_code == 304
|
||||
assert repeated.content == b""
|
||||
assert repeated.headers["ETag"] == response.headers["ETag"]
|
||||
assert admin_client.get("/admin/agent/opencode/path", **headers).headers["Cache-Control"] == "no-store"
|
||||
|
||||
|
||||
def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
|
||||
response = admin_client.get(
|
||||
"/admin/agent/opencode/global/event",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.streaming
|
||||
assert response.is_async
|
||||
assert response.headers["X-Accel-Buffering"] == "no"
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["event", "global/event"])
|
||||
def test_opencode_proxy_sse_delivers_first_event_immediately(admin_client, live_opencode, path):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
# The real /event endpoint emits heartbeats indefinitely. A bounded
|
||||
# upstream read exposes accidental buffering without hanging the test.
|
||||
status, headers, body = runtime.proxy(
|
||||
{**live_opencode.settings, "timeout": 1},
|
||||
"GET",
|
||||
path,
|
||||
(),
|
||||
{},
|
||||
b"",
|
||||
)
|
||||
assert status == 200
|
||||
assert not isinstance(body, bytes)
|
||||
assert headers["Content-Type"] == "text/event-stream"
|
||||
asyncio.run(body.aclose())
|
||||
response = admin_client.get(
|
||||
f"/admin/agent/opencode/{path}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
async def first_event():
|
||||
stream = response.streaming_content
|
||||
try:
|
||||
async with asyncio.timeout(2):
|
||||
return await anext(stream)
|
||||
finally:
|
||||
await stream.aclose()
|
||||
|
||||
assert b"server.connected" in asyncio.run(first_event())
|
||||
|
||||
|
||||
def test_opencode_proxy_sse_returns_headers_before_restart_finishes(admin_client, live_opencode):
|
||||
from archivebox.core.asgi import application
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
from django.conf import settings as django_settings
|
||||
|
||||
owned_process = runtime._PROCESS
|
||||
assert owned_process is not None
|
||||
session_cookie_name = django_settings.SESSION_COOKIE_NAME
|
||||
session_cookie = admin_client.cookies[session_cookie_name].value
|
||||
path = "/admin/agent/opencode/global/event"
|
||||
|
||||
async def request_event_stream():
|
||||
communicator = ApplicationCommunicator(
|
||||
application,
|
||||
{
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0"},
|
||||
"http_version": "1.1",
|
||||
"method": "GET",
|
||||
"scheme": "http",
|
||||
"path": path,
|
||||
"raw_path": path.encode(),
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"host", ADMIN_TEST_HOST.encode()),
|
||||
(b"cookie", f"{session_cookie_name}={session_cookie}".encode()),
|
||||
(b"sec-fetch-site", b"same-origin"),
|
||||
],
|
||||
"client": ("127.0.0.1", 12345),
|
||||
"server": ("127.0.0.1", 5797),
|
||||
},
|
||||
)
|
||||
runtime._PROCESS_LOCK.acquire()
|
||||
runtime._PROCESS = None
|
||||
try:
|
||||
await communicator.send_input({"type": "http.request", "body": b"", "more_body": False})
|
||||
response_start = await communicator.receive_output(timeout=2)
|
||||
assert response_start["type"] == "http.response.start"
|
||||
assert response_start["status"] == 200
|
||||
finally:
|
||||
try:
|
||||
await communicator.send_input({"type": "http.disconnect"})
|
||||
await communicator.wait(timeout=5)
|
||||
finally:
|
||||
runtime._PROCESS = owned_process
|
||||
runtime._PROCESS_LOCK.release()
|
||||
await asyncio.get_running_loop().shutdown_default_executor()
|
||||
|
||||
asyncio.run(request_event_stream())
|
||||
assert runtime._PROCESS is owned_process
|
||||
assert owned_process.poll() is None
|
||||
|
||||
|
||||
def test_opencode_starts_with_isolated_state(admin_client, live_opencode):
|
||||
workdir = str(live_opencode.config.data_dir.resolve())
|
||||
state_dir = live_opencode.config.state_dir
|
||||
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert agent.status_code == 200
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
frame = admin_client.get(
|
||||
runtime._project_route(live_opencode.config.data_dir),
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
)
|
||||
assert frame.status_code == 302
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
|
||||
project = requests.get(
|
||||
f"{live_opencode.settings['origin']}/project/current",
|
||||
params={"directory": workdir},
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
project.raise_for_status()
|
||||
|
||||
config = requests.get(
|
||||
f"{live_opencode.settings['origin']}/global/config",
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
config.raise_for_status()
|
||||
|
||||
assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
|
||||
assert project.json()["id"] == "global"
|
||||
assert not project.json().get("vcs")
|
||||
assert not config.json().get("model")
|
||||
assert config.json()["snapshot"] is False
|
||||
assert live_opencode.process.poll() is None
|
||||
path = requests.get(
|
||||
f"{live_opencode.settings['origin']}/path",
|
||||
params={"directory": workdir},
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
path.raise_for_status()
|
||||
assert Path(path.json()["directory"]).resolve() == Path(workdir)
|
||||
|
||||
diff = requests.get(
|
||||
f"{live_opencode.settings['origin']}/vcs/diff",
|
||||
params={"directory": workdir, "mode": "git"},
|
||||
timeout=5,
|
||||
)
|
||||
diff.raise_for_status()
|
||||
assert diff.json() == []
|
||||
assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
|
||||
assert (state_dir / "SKILL.md").is_file()
|
||||
assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
|
||||
|
||||
|
||||
def test_opencode_invalid_state_does_not_break_archivebox(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
runtime._stop_owned_process()
|
||||
invalid_state = live_opencode.config.state_dir / "config"
|
||||
invalid_state.rename(live_opencode.config.state_dir / "saved-config")
|
||||
invalid_state.write_text("Preserve this file.")
|
||||
|
||||
wrapper = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert wrapper.status_code == 200
|
||||
for url in (
|
||||
runtime._project_route(live_opencode.config.data_dir),
|
||||
"/admin/agent/opencode/global/health",
|
||||
):
|
||||
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 503
|
||||
assert str(invalid_state).encode() not in response.content
|
||||
|
||||
stream = admin_client.get("/admin/agent/opencode/global/event", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert stream.status_code == 200
|
||||
|
||||
async def read_failure():
|
||||
return b"".join([chunk async for chunk in stream.streaming_content])
|
||||
|
||||
assert asyncio.run(read_failure()) == b'event: error\ndata: {"error":"OpenCode unavailable"}\n\n'
|
||||
|
||||
for url in ("/health/", "/add/", "/admin/core/snapshot/"):
|
||||
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200
|
||||
assert invalid_state.read_text() == "Preserve this file."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("damaged_file", "missing"),
|
||||
[
|
||||
("runtime.py", True),
|
||||
("templates/agent.html", True),
|
||||
("templates/agent.html", False),
|
||||
("templates/navigation.html", False),
|
||||
("templates/add.html", False),
|
||||
],
|
||||
)
|
||||
def test_opencode_incomplete_install_does_not_break_archivebox(installed_opencode, tmp_path, damaged_file, missing):
|
||||
import abx_plugins
|
||||
|
||||
# Exercise a genuinely incomplete installation in a separate process;
|
||||
# never alter the shared package or intercept Python imports.
|
||||
site = tmp_path / "site"
|
||||
installed = site / "abx_plugins"
|
||||
shutil.copytree(Path(abx_plugins.__file__).parent, installed, ignore=shutil.ignore_patterns("__pycache__", "tests"))
|
||||
damaged_path = installed / "plugins" / "opencode" / damaged_file
|
||||
if missing:
|
||||
damaged_path.unlink()
|
||||
else:
|
||||
damaged_path.write_text("{% invalid_template_tag %}")
|
||||
expected_status = 200 if damaged_file in {"templates/navigation.html", "templates/add.html"} else 503
|
||||
script = f"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import abx_plugins
|
||||
from django.test import Client
|
||||
from django.contrib.auth import get_user_model
|
||||
assert Path(abx_plugins.__file__).is_relative_to({str(site)!r})
|
||||
user = get_user_model().objects.create_superuser(username='optional-service-test')
|
||||
client = Client(HTTP_HOST={ADMIN_TEST_HOST!r})
|
||||
client.force_login(user)
|
||||
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
|
||||
assert client.get(path).status_code == 200, path
|
||||
assert 'abx_plugins.plugins.opencode.runtime' not in sys.modules
|
||||
response = client.get('/admin/agent')
|
||||
assert response.status_code == {expected_status}, response.status_code
|
||||
if response.status_code == 503:
|
||||
assert response.content == b'AI service unavailable. See server logs.'
|
||||
if {damaged_file != "runtime.py"!r}:
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
assert runtime._PROCESS is None
|
||||
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
|
||||
assert client.get(path).status_code == 200, path
|
||||
print('OPTIONAL_SERVICE_FAILURE_ISOLATED')
|
||||
"""
|
||||
result = run_archivebox_cmd(
|
||||
["shell", "-c", script],
|
||||
cwd=installed_opencode.config.data_dir,
|
||||
env={**installed_opencode.config.env, "PYTHONPATH": str(site)},
|
||||
timeout=90,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "OPTIONAL_SERVICE_FAILURE_ISOLATED" in result.stdout
|
||||
@@ -0,0 +1,291 @@
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
def _reset_runtime_config() -> None:
|
||||
from archivebox.config import common
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
for value in vars(common).values():
|
||||
cache_clear = getattr(value, "cache_clear", None)
|
||||
if cache_clear is not None:
|
||||
cache_clear()
|
||||
Machine.current()
|
||||
|
||||
|
||||
def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
|
||||
os.chdir(data_dir)
|
||||
result = run_archivebox_cmd(
|
||||
["config", "--set", *values],
|
||||
cwd=data_dir,
|
||||
env=env,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
_reset_runtime_config()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opencode_archive_config(initialized_archive):
|
||||
port = _free_port()
|
||||
state_dir = initialized_archive / "opencode"
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
|
||||
"OPENCODE_ENABLED": "True",
|
||||
"OPENCODE_HOST": "127.0.0.1",
|
||||
"OPENCODE_PORT": str(port),
|
||||
"OPENCODE_WORKDIR": str(initialized_archive),
|
||||
"OPENCODE_STATE_DIR": str(state_dir),
|
||||
"OPENCODE_TIMEOUT": "60",
|
||||
},
|
||||
)
|
||||
_set_archivebox_config(
|
||||
initialized_archive,
|
||||
"OPENCODE_ENABLED=True",
|
||||
"OPENCODE_HOST=127.0.0.1",
|
||||
f"OPENCODE_PORT={port}",
|
||||
f"OPENCODE_WORKDIR={initialized_archive}",
|
||||
f"OPENCODE_STATE_DIR={state_dir}",
|
||||
"OPENCODE_TIMEOUT=60",
|
||||
env=env,
|
||||
)
|
||||
return SimpleNamespace(data_dir=initialized_archive, port=port, state_dir=state_dir, env=env)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def installed_opencode(opencode_archive_config):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
install = run_archivebox_cmd(
|
||||
["install", "opencode", "--binproviders=env,pnpm"],
|
||||
cwd=opencode_archive_config.data_dir,
|
||||
env=opencode_archive_config.env,
|
||||
timeout=1200,
|
||||
)
|
||||
assert install.returncode == 0, install.stderr or install.stdout
|
||||
_reset_runtime_config()
|
||||
|
||||
config = get_config().model_dump(mode="json")
|
||||
settings = runtime._settings(config, opencode_archive_config.data_dir)
|
||||
settings["archivebox_base_url"] = "http://admin.archivebox.localhost:5797"
|
||||
settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:5797/admin"
|
||||
settings["archivebox_api_url"] = "http://admin.archivebox.localhost:5797/api/"
|
||||
binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"])
|
||||
version = binary.exec(
|
||||
cmd=("--version",),
|
||||
env={**os.environ, **binary_env},
|
||||
timeout=120,
|
||||
)
|
||||
assert version.returncode == 0, version.stderr or version.stdout
|
||||
return SimpleNamespace(config=opencode_archive_config, settings=settings)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_opencode(installed_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
settings = installed_opencode.settings
|
||||
ok, error = runtime._ensure_opencode(settings)
|
||||
assert ok, error
|
||||
|
||||
process = runtime._PROCESS
|
||||
assert process is not None
|
||||
try:
|
||||
yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process)
|
||||
finally:
|
||||
runtime._stop_owned_process()
|
||||
|
||||
|
||||
def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive):
|
||||
_set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False")
|
||||
|
||||
assert get_config().OPENCODE_ENABLED is False
|
||||
assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404
|
||||
for path in ("/add/", "/admin/core/snapshot/"):
|
||||
response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200
|
||||
assert b'href="/admin/agent"' not in response.content
|
||||
|
||||
|
||||
def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_opencode):
|
||||
response = admin_client.post(
|
||||
"/admin/agent/opencode/session",
|
||||
data=b"{}",
|
||||
content_type="application/json",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="cross-site",
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_proxy_waits_for_owned_process_readiness(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
process = runtime._PROCESS
|
||||
assert process is not None
|
||||
runtime._PROCESS_READY = None
|
||||
workdir = quote(str(live_opencode.config.data_dir.resolve()))
|
||||
|
||||
response = admin_client.get(
|
||||
f"/admin/agent/opencode/path?directory={workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert runtime._PROCESS is process
|
||||
assert runtime._PROCESS_READY is process
|
||||
|
||||
|
||||
def test_opencode_proxy_preserves_protocol_headers(admin_client, live_opencode):
|
||||
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
|
||||
created = admin_client.post(
|
||||
"/admin/agent/opencode/pty",
|
||||
data={"command": "/bin/sh", "args": []},
|
||||
content_type="application/json",
|
||||
**headers,
|
||||
)
|
||||
assert created.status_code == 200
|
||||
path = f"/admin/agent/opencode/pty/{created.json()['id']}"
|
||||
try:
|
||||
response = admin_client.post(path + "/connect-token", HTTP_X_OPENCODE_TICKET="1", **headers)
|
||||
assert response.status_code == 200, response.content
|
||||
assert response.json()["ticket"]
|
||||
finally:
|
||||
deleted = admin_client.delete(path, **headers)
|
||||
assert deleted.status_code == 200, deleted.content
|
||||
|
||||
|
||||
def test_opencode_starts_with_isolated_state(admin_client, live_opencode):
|
||||
workdir = str(live_opencode.config.data_dir.resolve())
|
||||
state_dir = live_opencode.config.state_dir
|
||||
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert agent.status_code == 200
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
frame = admin_client.get(
|
||||
runtime._project_route(live_opencode.config.data_dir),
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
)
|
||||
assert frame.status_code == 302
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
|
||||
project = requests.get(
|
||||
f"{live_opencode.settings['origin']}/project/current",
|
||||
params={"directory": workdir},
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
project.raise_for_status()
|
||||
|
||||
config = requests.get(
|
||||
f"{live_opencode.settings['origin']}/global/config",
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
config.raise_for_status()
|
||||
|
||||
assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
|
||||
assert project.json()["id"] == "global"
|
||||
assert not project.json().get("vcs")
|
||||
assert not config.json().get("model")
|
||||
assert config.json()["snapshot"] is False
|
||||
assert live_opencode.process.poll() is None
|
||||
path = requests.get(
|
||||
f"{live_opencode.settings['origin']}/path",
|
||||
params={"directory": workdir},
|
||||
timeout=live_opencode.settings["timeout"],
|
||||
)
|
||||
path.raise_for_status()
|
||||
assert Path(path.json()["directory"]).resolve() == Path(workdir)
|
||||
|
||||
diff = requests.get(
|
||||
f"{live_opencode.settings['origin']}/vcs/diff",
|
||||
params={"directory": workdir, "mode": "git"},
|
||||
timeout=5,
|
||||
)
|
||||
diff.raise_for_status()
|
||||
assert diff.json() == []
|
||||
assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
|
||||
assert (state_dir / "SKILL.md").is_file()
|
||||
assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("damaged_file", "missing"),
|
||||
[
|
||||
("runtime.py", True),
|
||||
("templates/agent.html", True),
|
||||
("templates/agent.html", False),
|
||||
("templates/navigation.html", False),
|
||||
("templates/add.html", False),
|
||||
],
|
||||
)
|
||||
def test_opencode_incomplete_install_does_not_break_archivebox(installed_opencode, tmp_path, damaged_file, missing):
|
||||
import abx_plugins
|
||||
|
||||
# Exercise a genuinely incomplete installation in a separate process;
|
||||
# never alter the shared package or intercept Python imports.
|
||||
site = tmp_path / "site"
|
||||
installed = site / "abx_plugins"
|
||||
shutil.copytree(Path(abx_plugins.__file__).parent, installed, ignore=shutil.ignore_patterns("__pycache__", "tests"))
|
||||
damaged_path = installed / "plugins" / "opencode" / damaged_file
|
||||
if missing:
|
||||
damaged_path.unlink()
|
||||
else:
|
||||
damaged_path.write_text("{% invalid_template_tag %}")
|
||||
expected_status = 200 if damaged_file in {"templates/navigation.html", "templates/add.html"} else 503
|
||||
script = f"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import abx_plugins
|
||||
from django.test import Client
|
||||
from django.contrib.auth import get_user_model
|
||||
assert Path(abx_plugins.__file__).is_relative_to({str(site)!r})
|
||||
user = get_user_model().objects.create_superuser(username='optional-service-test')
|
||||
client = Client(HTTP_HOST={ADMIN_TEST_HOST!r})
|
||||
client.force_login(user)
|
||||
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
|
||||
assert client.get(path).status_code == 200, path
|
||||
assert 'abx_plugins.plugins.opencode.runtime' not in sys.modules
|
||||
response = client.get('/admin/agent')
|
||||
assert response.status_code == {expected_status}, response.status_code
|
||||
if response.status_code == 503:
|
||||
assert response.content == b'AI service unavailable. See server logs.'
|
||||
if {damaged_file != "runtime.py"!r}:
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
assert runtime._PROCESS is None
|
||||
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
|
||||
assert client.get(path).status_code == 200, path
|
||||
print('OPTIONAL_SERVICE_FAILURE_ISOLATED')
|
||||
"""
|
||||
result = run_archivebox_cmd(
|
||||
["shell", "-c", script],
|
||||
cwd=installed_opencode.config.data_dir,
|
||||
env={**installed_opencode.config.env, "PYTHONPATH": str(site)},
|
||||
timeout=90,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "OPTIONAL_SERVICE_FAILURE_ISOLATED" in result.stdout
|
||||
@@ -0,0 +1,251 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FutureTimeoutError
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from asgiref.testing import ApplicationCommunicator
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST
|
||||
from archivebox.config.common import get_config
|
||||
|
||||
|
||||
from .test_opencode_agent_1 import (
|
||||
pytestmark as pytestmark,
|
||||
_free_port as _free_port,
|
||||
_reset_runtime_config as _reset_runtime_config,
|
||||
_set_archivebox_config as _set_archivebox_config,
|
||||
opencode_archive_config as opencode_archive_config,
|
||||
installed_opencode as installed_opencode,
|
||||
live_opencode as live_opencode,
|
||||
)
|
||||
|
||||
|
||||
def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
os.chdir(initialized_archive)
|
||||
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
|
||||
_reset_runtime_config()
|
||||
assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False
|
||||
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
|
||||
assert response.status_code == 404
|
||||
assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None
|
||||
|
||||
|
||||
def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode):
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 302
|
||||
assert "/admin/login/" in response.headers["Location"]
|
||||
|
||||
next_path = "/admin/agent?x=1&next=https://example.com"
|
||||
response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 302
|
||||
assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]}
|
||||
|
||||
user = django_user_model.objects.create_user(username="regular", password="testpassword")
|
||||
client.force_login(user)
|
||||
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
frame_path = runtime._project_route(live_opencode.config.data_dir)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert f'<iframe src="{frame_path}"'.encode() in response.content
|
||||
assert b'id="opencode-agent-welcome"' in response.content
|
||||
assert b'id="header"' in response.content
|
||||
assert b'id="progress-monitor"' in response.content
|
||||
assert b'<a href="/admin/agent" class="navbar-item navbar-ai">' in response.content
|
||||
add_page = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert add_page.status_code == 200
|
||||
assert '<a href="/admin/agent">💬 Crawl with AI</a>'.encode() in add_page.content
|
||||
assert response.context["proxy_prefix"] == runtime._PROXY_PREFIX
|
||||
assert b"/_archivebox/health" not in response.content
|
||||
assert b"window.setInterval(check, 3000)" not in response.content
|
||||
assert response.headers["X-Frame-Options"] == "DENY"
|
||||
assert response.headers["Content-Security-Policy"] == "frame-ancestors 'none'"
|
||||
|
||||
frame = admin_client.get(
|
||||
frame_path,
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert frame.status_code == 302
|
||||
assert frame.headers["Location"].startswith(frame_path + "/")
|
||||
session = admin_client.get(
|
||||
frame.headers["Location"],
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert session.status_code == 200
|
||||
assert session.headers["X-Frame-Options"] == "SAMEORIGIN"
|
||||
assert session.headers["Content-Security-Policy"] == "frame-ancestors 'self'"
|
||||
|
||||
|
||||
def test_opencode_oauth_callback_waits_for_user_and_preserves_cancellation(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
settings = {**live_opencode.settings, "timeout": 1}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
status, _, body = runtime.proxy(settings, "POST", "provider/openai/oauth/authorize", (), headers, b'{"method":0}')
|
||||
assert status == 200
|
||||
authorization = json.loads(body)
|
||||
assert urlsplit(authorization["url"]).hostname == "auth.openai.com"
|
||||
redirect = parse_qs(urlsplit(authorization["url"]).query)["redirect_uri"][0]
|
||||
callback_origin = urlsplit(redirect)
|
||||
assert callback_origin.hostname == "localhost"
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
callback = executor.submit(runtime.proxy, settings, "POST", "provider/openai/oauth/callback", (), headers, b'{"method":0}')
|
||||
try:
|
||||
# A real pending authorization must outlive the ordinary API read
|
||||
# timeout. No tokens or substituted provider responses are used.
|
||||
with pytest.raises(FutureTimeoutError):
|
||||
callback.result(timeout=2)
|
||||
finally:
|
||||
cancelled = requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=5)
|
||||
assert cancelled.status_code == 200
|
||||
assert cancelled.text == "Login cancelled"
|
||||
status, response_headers, body = callback.result(timeout=5)
|
||||
assert status == 500
|
||||
assert response_headers["Content-Type"].startswith("application/json")
|
||||
error = json.loads(body)
|
||||
assert error["name"] == "UnknownError"
|
||||
assert error["data"]["ref"].startswith("err_")
|
||||
runtime._stop_owned_process()
|
||||
# The pnpm launcher can exit before its server child. Stopping the owned
|
||||
# process must release both listeners so another collection can authorize.
|
||||
with pytest.raises(requests.ConnectionError):
|
||||
requests.get(settings["origin"] + "/global/health", timeout=2)
|
||||
with pytest.raises(requests.ConnectionError):
|
||||
requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=2)
|
||||
|
||||
|
||||
def test_opencode_static_assets_cache_privately_and_revalidate(admin_client, live_opencode):
|
||||
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
|
||||
page = admin_client.get("/admin/agent/opencode/", **headers)
|
||||
assert page.status_code == 200
|
||||
asset = re.search(rb'src="(/admin/agent/opencode/assets/[^\"]+\.js)"', page.content)
|
||||
assert asset is not None
|
||||
path = asset[1].decode()
|
||||
response = admin_client.get(path, **headers)
|
||||
assert response.status_code == 200
|
||||
assert "private" in response.headers["Cache-Control"]
|
||||
assert "max-age=" in response.headers["Cache-Control"]
|
||||
assert b"/admin/agent/opencode" in response.content
|
||||
repeated = admin_client.get(path, HTTP_IF_NONE_MATCH=response.headers["ETag"], **headers)
|
||||
assert repeated.status_code == 304
|
||||
assert repeated.content == b""
|
||||
assert repeated.headers["ETag"] == response.headers["ETag"]
|
||||
assert admin_client.get("/admin/agent/opencode/path", **headers).headers["Cache-Control"] == "no-store"
|
||||
|
||||
|
||||
def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
|
||||
response = admin_client.get(
|
||||
"/admin/agent/opencode/global/event",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.streaming
|
||||
assert response.is_async
|
||||
assert response.headers["X-Accel-Buffering"] == "no"
|
||||
assert response.headers["Cache-Control"] == "no-store"
|
||||
|
||||
|
||||
def test_opencode_proxy_sse_returns_headers_before_restart_finishes(admin_client, live_opencode):
|
||||
from archivebox.core.asgi import application
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
from django.conf import settings as django_settings
|
||||
|
||||
owned_process = runtime._PROCESS
|
||||
assert owned_process is not None
|
||||
session_cookie_name = django_settings.SESSION_COOKIE_NAME
|
||||
session_cookie = admin_client.cookies[session_cookie_name].value
|
||||
path = "/admin/agent/opencode/global/event"
|
||||
|
||||
async def request_event_stream():
|
||||
communicator = ApplicationCommunicator(
|
||||
application,
|
||||
{
|
||||
"type": "http",
|
||||
"asgi": {"version": "3.0"},
|
||||
"http_version": "1.1",
|
||||
"method": "GET",
|
||||
"scheme": "http",
|
||||
"path": path,
|
||||
"raw_path": path.encode(),
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"host", ADMIN_TEST_HOST.encode()),
|
||||
(b"cookie", f"{session_cookie_name}={session_cookie}".encode()),
|
||||
(b"sec-fetch-site", b"same-origin"),
|
||||
],
|
||||
"client": ("127.0.0.1", 12345),
|
||||
"server": ("127.0.0.1", 5797),
|
||||
},
|
||||
)
|
||||
runtime._PROCESS_LOCK.acquire()
|
||||
runtime._PROCESS = None
|
||||
try:
|
||||
await communicator.send_input({"type": "http.request", "body": b"", "more_body": False})
|
||||
response_start = await communicator.receive_output(timeout=2)
|
||||
assert response_start["type"] == "http.response.start"
|
||||
assert response_start["status"] == 200
|
||||
finally:
|
||||
try:
|
||||
await communicator.send_input({"type": "http.disconnect"})
|
||||
await communicator.wait(timeout=5)
|
||||
finally:
|
||||
runtime._PROCESS = owned_process
|
||||
runtime._PROCESS_LOCK.release()
|
||||
await asyncio.get_running_loop().shutdown_default_executor()
|
||||
|
||||
asyncio.run(request_event_stream())
|
||||
assert runtime._PROCESS is owned_process
|
||||
assert owned_process.poll() is None
|
||||
|
||||
|
||||
def test_opencode_invalid_state_does_not_break_archivebox(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
runtime._stop_owned_process()
|
||||
invalid_state = live_opencode.config.state_dir / "config"
|
||||
invalid_state.rename(live_opencode.config.state_dir / "saved-config")
|
||||
invalid_state.write_text("Preserve this file.")
|
||||
|
||||
wrapper = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert wrapper.status_code == 200
|
||||
for url in (
|
||||
runtime._project_route(live_opencode.config.data_dir),
|
||||
"/admin/agent/opencode/global/health",
|
||||
):
|
||||
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 503
|
||||
assert str(invalid_state).encode() not in response.content
|
||||
|
||||
stream = admin_client.get("/admin/agent/opencode/global/event", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert stream.status_code == 200
|
||||
|
||||
async def read_failure():
|
||||
return b"".join([chunk async for chunk in stream.streaming_content])
|
||||
|
||||
assert asyncio.run(read_failure()) == b'event: error\ndata: {"error":"OpenCode unavailable"}\n\n'
|
||||
|
||||
for url in ("/health/", "/add/", "/admin/core/snapshot/"):
|
||||
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200
|
||||
assert invalid_state.read_text() == "Preserve this file."
|
||||
@@ -0,0 +1,191 @@
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST
|
||||
|
||||
|
||||
from .test_opencode_agent_1 import (
|
||||
pytestmark as pytestmark,
|
||||
_free_port as _free_port,
|
||||
_reset_runtime_config as _reset_runtime_config,
|
||||
_set_archivebox_config as _set_archivebox_config,
|
||||
opencode_archive_config as opencode_archive_config,
|
||||
installed_opencode as installed_opencode,
|
||||
live_opencode as live_opencode,
|
||||
)
|
||||
|
||||
|
||||
def test_opencode_proxy_blocks_cross_origin_mutation(admin_client, db, live_opencode):
|
||||
response = admin_client.post(
|
||||
"/admin/agent/opencode/session",
|
||||
data=b"{}",
|
||||
content_type="application/json",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_ORIGIN="https://evil.example",
|
||||
)
|
||||
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_opencode_cold_agent_wrapper_returns_before_server_starts(admin_client, installed_opencode):
|
||||
import time
|
||||
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
assert not runtime._owned_process_running()
|
||||
started = time.monotonic()
|
||||
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert time.monotonic() - started < 3
|
||||
assert not runtime._owned_process_running()
|
||||
assert b'id="opencode-agent-welcome"' in response.content
|
||||
assert f'<iframe src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
|
||||
|
||||
|
||||
def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):
|
||||
workdir = str(live_opencode.config.data_dir.resolve())
|
||||
encoded_workdir = quote(workdir)
|
||||
|
||||
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert agent.status_code == 200
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
frame_path = runtime._project_route(live_opencode.config.data_dir)
|
||||
frame = admin_client.get(frame_path, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert frame.status_code == 302
|
||||
session_id = frame.headers["Location"].rsplit("/", 1)[-1]
|
||||
|
||||
project = admin_client.get(
|
||||
f"/admin/agent/opencode/project/current?directory={encoded_workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert project.status_code == 200
|
||||
assert project.json()["id"] == "global"
|
||||
assert not project.json().get("vcs")
|
||||
|
||||
path = admin_client.get(
|
||||
f"/admin/agent/opencode/path?directory={encoded_workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert path.status_code == 200
|
||||
assert path.json()["directory"] == workdir
|
||||
|
||||
sessions = admin_client.get(
|
||||
f"/admin/agent/opencode/session?directory={encoded_workdir}&roots=true&limit=55",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert sessions.status_code == 200
|
||||
assert any(session["id"] == session_id and session["directory"] == workdir for session in sessions.json())
|
||||
assert not (Path(workdir) / ".git").exists()
|
||||
|
||||
|
||||
def test_opencode_proxy_restarts_server_for_an_existing_agent_page(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
old_process = runtime._PROCESS
|
||||
runtime._stop_owned_process()
|
||||
|
||||
response = admin_client.get(
|
||||
"/admin/agent/opencode/global/health",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert runtime._PROCESS is not None
|
||||
assert runtime._PROCESS is not old_process
|
||||
assert runtime._PROCESS.poll() is None
|
||||
|
||||
|
||||
def test_concurrent_opencode_startup_waits_until_server_is_ready(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
runtime._stop_owned_process()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(runtime._ensure_opencode, [live_opencode.settings] * 2))
|
||||
|
||||
assert results == [(True, ""), (True, "")]
|
||||
assert runtime._health(live_opencode.settings)
|
||||
|
||||
|
||||
def test_opencode_does_not_probe_or_replace_a_ready_owned_process(live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
process = runtime._PROCESS
|
||||
settings = {**live_opencode.settings, "port": _free_port()}
|
||||
settings["origin"] = f"http://{settings['host']}:{settings['port']}"
|
||||
|
||||
ok, error = runtime._ensure_opencode(settings)
|
||||
|
||||
assert ok, error
|
||||
assert process is not None
|
||||
assert runtime._PROCESS is process
|
||||
assert process.poll() is None
|
||||
|
||||
|
||||
def test_opencode_proxy_does_not_wait_for_recovery_lock(admin_client, live_opencode):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
workdir = quote(str(live_opencode.config.data_dir.resolve()))
|
||||
assert runtime._owned_process_ready()
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
runtime._PROCESS_LOCK.acquire()
|
||||
try:
|
||||
request = executor.submit(
|
||||
admin_client.get,
|
||||
f"/admin/agent/opencode/path?directory={workdir}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
response = request.result(timeout=5)
|
||||
finally:
|
||||
runtime._PROCESS_LOCK.release()
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert str(live_opencode.config.data_dir.resolve()).encode() in response.content
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["event", "global/event"])
|
||||
def test_opencode_proxy_sse_delivers_first_event_immediately(admin_client, live_opencode, path):
|
||||
from abx_plugins.plugins.opencode import runtime
|
||||
|
||||
# The real /event endpoint emits heartbeats indefinitely. A bounded
|
||||
# upstream read exposes accidental buffering without hanging the test.
|
||||
status, headers, body = runtime.proxy(
|
||||
{**live_opencode.settings, "timeout": 1},
|
||||
"GET",
|
||||
path,
|
||||
(),
|
||||
{},
|
||||
b"",
|
||||
)
|
||||
assert status == 200
|
||||
assert not isinstance(body, bytes)
|
||||
assert headers["Content-Type"] == "text/event-stream"
|
||||
asyncio.run(body.aclose())
|
||||
response = admin_client.get(
|
||||
f"/admin/agent/opencode/{path}",
|
||||
HTTP_HOST=ADMIN_TEST_HOST,
|
||||
HTTP_SEC_FETCH_SITE="same-origin",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
async def first_event():
|
||||
stream = response.streaming_content
|
||||
try:
|
||||
async with asyncio.timeout(2):
|
||||
return await anext(stream)
|
||||
finally:
|
||||
await stream.aclose()
|
||||
|
||||
assert b"server.connected" in asyncio.run(first_event())
|
||||
+52
-264
@@ -1,6 +1,5 @@
|
||||
"""Exercise the real agent wrapper with Chromium's native storage failures."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -10,16 +9,64 @@ import pytest
|
||||
import requests
|
||||
|
||||
from .conftest import get_free_port, run_archivebox_cmd, start_archivebox_server, stop_archivebox_process
|
||||
from .test_opencode_agent import _set_archivebox_config
|
||||
from .test_opencode_agent import installed_opencode as installed_opencode
|
||||
from .test_opencode_agent import live_opencode as live_opencode
|
||||
from .test_opencode_agent import opencode_archive_config as opencode_archive_config
|
||||
from .test_opencode_agent_1 import _set_archivebox_config
|
||||
from .test_opencode_agent_1 import installed_opencode as installed_opencode
|
||||
from .test_opencode_agent_1 import live_opencode as live_opencode
|
||||
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
|
||||
from .test_server_security_browser import browser_runtime as browser_runtime
|
||||
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent_server(installed_opencode, browser_runtime, request):
|
||||
port = get_free_port()
|
||||
mode, subdomains = getattr(request, "param", ("safe-onedomain-nojsreplay", False))
|
||||
base_url = f"http://archivebox.localhost:{port}" if subdomains else f"http://localhost:{port}"
|
||||
url = f"http://admin.archivebox.localhost:{port}" if subdomains else base_url
|
||||
config = installed_opencode.config
|
||||
_set_archivebox_config(
|
||||
config.data_dir,
|
||||
f"BASE_URL={base_url}",
|
||||
f"SERVER_SECURITY_MODE={mode}",
|
||||
)
|
||||
user = run_archivebox_cmd(
|
||||
[
|
||||
"shell",
|
||||
"-c",
|
||||
"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(username='agent-browser-test', password='test-password'); User.objects.create_user(username='agent-regular-test', password='test-password', is_staff=True)",
|
||||
],
|
||||
cwd=config.data_dir,
|
||||
env=config.env,
|
||||
)
|
||||
assert user.returncode == 0, user.stderr or user.stdout
|
||||
process = start_archivebox_server(config.data_dir, port=port, env=config.env, log_name="agent-browser-server.log")
|
||||
try:
|
||||
yield url, config.data_dir, process
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
stop_archivebox_process(process)
|
||||
|
||||
|
||||
def _login_cookie(server_url, username="agent-browser-test"):
|
||||
session = requests.Session()
|
||||
login_url = server_url + "/admin/login/"
|
||||
page = session.get(login_url, timeout=10)
|
||||
assert page.status_code == 200
|
||||
token = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', page.text)
|
||||
assert token is not None
|
||||
response = session.post(
|
||||
login_url,
|
||||
data={"username": username, "password": "test-password", "csrfmiddlewaretoken": token[1]},
|
||||
headers={"Referer": login_url},
|
||||
allow_redirects=False,
|
||||
timeout=10,
|
||||
)
|
||||
assert response.status_code == 302
|
||||
return "; ".join(f"{key}={value}" for key, value in session.cookies.items())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"agent_server",
|
||||
[
|
||||
@@ -231,262 +278,3 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "AGENT_NAVIGATION_OK" in result.stdout
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent_server(installed_opencode, browser_runtime, request):
|
||||
port = get_free_port()
|
||||
mode, subdomains = getattr(request, "param", ("safe-onedomain-nojsreplay", False))
|
||||
base_url = f"http://archivebox.localhost:{port}" if subdomains else f"http://localhost:{port}"
|
||||
url = f"http://admin.archivebox.localhost:{port}" if subdomains else base_url
|
||||
config = installed_opencode.config
|
||||
_set_archivebox_config(
|
||||
config.data_dir,
|
||||
f"BASE_URL={base_url}",
|
||||
f"SERVER_SECURITY_MODE={mode}",
|
||||
)
|
||||
user = run_archivebox_cmd(
|
||||
[
|
||||
"shell",
|
||||
"-c",
|
||||
"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(username='agent-browser-test', password='test-password'); User.objects.create_user(username='agent-regular-test', password='test-password', is_staff=True)",
|
||||
],
|
||||
cwd=config.data_dir,
|
||||
env=config.env,
|
||||
)
|
||||
assert user.returncode == 0, user.stderr or user.stdout
|
||||
process = start_archivebox_server(config.data_dir, port=port, env=config.env, log_name="agent-browser-server.log")
|
||||
try:
|
||||
yield url, config.data_dir, process
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
stop_archivebox_process(process)
|
||||
|
||||
|
||||
def _login_cookie(server_url, username="agent-browser-test"):
|
||||
session = requests.Session()
|
||||
login_url = server_url + "/admin/login/"
|
||||
page = session.get(login_url, timeout=10)
|
||||
assert page.status_code == 200
|
||||
token = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', page.text)
|
||||
assert token is not None
|
||||
response = session.post(
|
||||
login_url,
|
||||
data={"username": username, "password": "test-password", "csrfmiddlewaretoken": token[1]},
|
||||
headers={"Referer": login_url},
|
||||
allow_redirects=False,
|
||||
timeout=10,
|
||||
)
|
||||
assert response.status_code == 302
|
||||
return "; ".join(f"{key}={value}" for key, value in session.cookies.items())
|
||||
|
||||
|
||||
def test_agent_websocket_rejects_unauthorized_access(agent_server):
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import InvalidStatus
|
||||
|
||||
server_url, _, _ = agent_server
|
||||
admin_cookie = _login_cookie(server_url)
|
||||
regular_cookie = _login_cookie(server_url, "agent-regular-test")
|
||||
|
||||
async def check_denials():
|
||||
for path, cookie, origin in (
|
||||
("/admin/agent/opencode/pty/unknown/connect", "", server_url),
|
||||
("/admin/agent/opencode/pty/unknown/connect", regular_cookie, server_url),
|
||||
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, "https://untrusted.example"),
|
||||
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, None),
|
||||
("/health/", admin_cookie, server_url),
|
||||
):
|
||||
with pytest.raises(InvalidStatus) as error:
|
||||
async with connect(
|
||||
server_url.replace("http", "ws", 1) + path,
|
||||
origin=origin,
|
||||
additional_headers={"Cookie": cookie},
|
||||
proxy=None,
|
||||
):
|
||||
pytest.fail("Unauthorized WebSocket was accepted")
|
||||
assert error.value.response.status_code == 403
|
||||
|
||||
asyncio.run(check_denials())
|
||||
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode,proxy_whitelist,use_cookie,http_status",
|
||||
[
|
||||
("unsafe-onedomain-noadmin", "", True, 403),
|
||||
("safe-onedomain-nojsreplay", "127.0.0.1/32", False, 200),
|
||||
("safe-onedomain-nojsreplay", "192.0.2.0/24", False, 302),
|
||||
],
|
||||
)
|
||||
def test_agent_websocket_matches_http_security_policy(agent_server, live_opencode, mode, proxy_whitelist, use_cookie, http_status):
|
||||
from urllib.parse import urlsplit
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import InvalidStatus
|
||||
|
||||
server_url, data_dir, process = agent_server
|
||||
cookie = _login_cookie(server_url)
|
||||
created = requests.post(
|
||||
server_url + "/admin/agent/opencode/pty",
|
||||
headers={"Cookie": cookie, "Origin": server_url},
|
||||
json={"command": "/bin/sh", "args": []},
|
||||
timeout=10,
|
||||
)
|
||||
assert created.status_code == 200
|
||||
pty_id = created.json()["id"]
|
||||
stop_archivebox_process(process)
|
||||
_set_archivebox_config(
|
||||
data_dir,
|
||||
f"SERVER_SECURITY_MODE={mode}",
|
||||
f"REVERSE_PROXY_WHITELIST={proxy_whitelist}",
|
||||
"REVERSE_PROXY_USER_HEADER=Remote-User",
|
||||
)
|
||||
restarted = start_archivebox_server(
|
||||
data_dir,
|
||||
port=urlsplit(server_url).port,
|
||||
env=live_opencode.config.env,
|
||||
log_name="agent-security-server.log",
|
||||
)
|
||||
headers = {"Cookie": cookie} if use_cookie else {"Remote-User": "agent-browser-test"}
|
||||
try:
|
||||
response = requests.get(server_url + "/admin/agent/opencode/global/health", headers=headers, allow_redirects=False, timeout=10)
|
||||
assert response.status_code == http_status
|
||||
|
||||
async def check_socket():
|
||||
connection = connect(
|
||||
server_url.replace("http", "ws", 1) + f"/admin/agent/opencode/pty/{pty_id}/connect",
|
||||
origin=server_url,
|
||||
additional_headers=headers,
|
||||
proxy=None,
|
||||
)
|
||||
if http_status != 200:
|
||||
with pytest.raises(InvalidStatus) as error:
|
||||
async with connection:
|
||||
pytest.fail("Disabled control plane or untrusted proxy accepted a WebSocket")
|
||||
assert error.value.response.status_code == 403
|
||||
return
|
||||
async with connection as socket, asyncio.timeout(10):
|
||||
await socket.send("printf 'ABX_%s\\n' PROXY_OK\n")
|
||||
output = ""
|
||||
while "ABX_PROXY_OK" not in output:
|
||||
chunk = await socket.recv()
|
||||
output += chunk.decode() if isinstance(chunk, bytes) else chunk
|
||||
assert "ABX_PROXY_OK" in output
|
||||
|
||||
asyncio.run(check_socket())
|
||||
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
|
||||
finally:
|
||||
deleted = requests.delete(live_opencode.settings["origin"] + f"/pty/{pty_id}", timeout=10)
|
||||
assert deleted.status_code == 200
|
||||
stop_archivebox_process(restarted)
|
||||
|
||||
|
||||
def test_agent_preserves_projects_and_survives_storage_failure(agent_server, browser_runtime):
|
||||
server_url, data_dir, _ = agent_server
|
||||
script = r"""
|
||||
const assert = require('node:assert/strict');
|
||||
const puppeteer = require('puppeteer');
|
||||
const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
|
||||
(async () => {
|
||||
for (const disabled of [false, true]) {
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: config.chrome,
|
||||
headless: true,
|
||||
// Headless interaction must not depend on the desktop display clock.
|
||||
args: ['--no-sandbox', '--disable-frame-rate-limit', ...(disabled ? ['--disable-local-storage'] : [])],
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.goto(new URL('/admin/login/', config.url).href, {waitUntil: 'domcontentloaded'});
|
||||
await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
|
||||
await page.locator('#login-form input[name="password"]').fill('test-password');
|
||||
assert.equal(await page.$eval('#login-form input[name="username"]', element => element.value), 'agent-browser-test');
|
||||
assert.equal(await page.$eval('#login-form input[name="password"]', element => element.value), 'test-password');
|
||||
await Promise.all([
|
||||
page.waitForNavigation({waitUntil: 'domcontentloaded'}),
|
||||
page.locator('#login-form input[type="submit"]').click(),
|
||||
]);
|
||||
assert.ok(!page.url().includes('/admin/login/'), page.url());
|
||||
assert.equal((await page.goto(config.url, {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
const welcome = '#opencode-agent-welcome';
|
||||
await page.waitForSelector(welcome, {visible: true});
|
||||
if (disabled) {
|
||||
assert.equal(await page.evaluate(() => {
|
||||
try { return localStorage === null; }
|
||||
catch (error) { if (error.name !== 'SecurityError') throw error; return true; }
|
||||
}), true);
|
||||
} else {
|
||||
await page.click('#opencode-agent-welcome-dismiss');
|
||||
await page.waitForSelector(welcome, {hidden: true});
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
assert.equal(await page.$eval(welcome, element => element.hidden), true);
|
||||
const existing = {
|
||||
list: [{type: 'http', http: {url: 'http://other.example'}}],
|
||||
projects: {
|
||||
local: [{worktree: '/other-project', expanded: false}],
|
||||
'http://other.example': [{worktree: '/remote-project', expanded: true}],
|
||||
},
|
||||
lastProject: {'http://other.example': '/remote-project'},
|
||||
};
|
||||
await page.evaluate(value => localStorage.setItem('opencode.global.dat:server', JSON.stringify(value)), existing);
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
await page.waitForFunction(workdir => {
|
||||
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
|
||||
return state.projects.local?.some(project => project.worktree === workdir);
|
||||
}, {}, config.workdir);
|
||||
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('opencode.global.dat:server')));
|
||||
assert.deepEqual(saved.list, existing.list);
|
||||
assert.deepEqual(saved.projects['http://other.example'], existing.projects['http://other.example']);
|
||||
assert.equal(saved.lastProject['http://other.example'], '/remote-project');
|
||||
const projects = saved.projects.local;
|
||||
assert.ok(Array.isArray(projects), JSON.stringify({url: page.url(), saved, existing}));
|
||||
assert.deepEqual(projects.find(project => project.worktree === '/other-project'), {worktree: '/other-project', expanded: false});
|
||||
assert.equal(projects.filter(project => project.worktree === config.workdir).length, 1);
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
assert.equal(await page.evaluate(workdir => {
|
||||
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
|
||||
return state.projects.local.filter(project => project.worktree === workdir).length;
|
||||
}, config.workdir), 1);
|
||||
|
||||
// Fill the actual browser quota; do not replace or intercept Storage methods.
|
||||
const failure = await page.evaluate(() => {
|
||||
localStorage.clear();
|
||||
let low = 0, high = 8 * 1024 * 1024;
|
||||
while (low < high) {
|
||||
const size = Math.ceil((low + high) / 2);
|
||||
try { localStorage.setItem('padding', 'x'.repeat(size)); low = size; }
|
||||
catch (error) { if (error.name !== 'QuotaExceededError') throw error; high = size - 1; }
|
||||
}
|
||||
try { localStorage.setItem('extra', '1'); return null; }
|
||||
catch (error) { return error.name; }
|
||||
});
|
||||
assert.equal(failure, 'QuotaExceededError');
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
await page.waitForSelector(welcome, {visible: true});
|
||||
}
|
||||
await page.click('#opencode-agent-welcome-dismiss');
|
||||
await page.waitForSelector(welcome, {hidden: true});
|
||||
assert.equal((await page.goto(new URL('/add/', config.url).href, {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
console.log('AGENT_STORAGE_FAILURE_ISOLATED');
|
||||
})().catch(error => { console.error(error); process.exitCode = 1; });
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[str(browser_runtime["node_binary"]), "-e", script],
|
||||
input=json.dumps(
|
||||
{
|
||||
"chrome": str(browser_runtime["chrome_binary"]),
|
||||
"url": f"{server_url}/admin/agent",
|
||||
"workdir": str(data_dir.resolve()),
|
||||
},
|
||||
),
|
||||
env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "AGENT_STORAGE_FAILURE_ISOLATED" in result.stdout
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Exercise the real agent wrapper with Chromium's native storage failures."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from .conftest import start_archivebox_server, stop_archivebox_process
|
||||
from .test_opencode_agent_1 import _set_archivebox_config
|
||||
from .test_opencode_agent_1 import installed_opencode as installed_opencode
|
||||
from .test_opencode_agent_1 import live_opencode as live_opencode
|
||||
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
|
||||
from .test_server_security_browser import browser_runtime as browser_runtime
|
||||
|
||||
|
||||
from .test_opencode_browser_1 import (
|
||||
pytestmark as pytestmark,
|
||||
agent_server as agent_server,
|
||||
_login_cookie as _login_cookie,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode,proxy_whitelist,use_cookie,http_status",
|
||||
[
|
||||
("unsafe-onedomain-noadmin", "", True, 403),
|
||||
("safe-onedomain-nojsreplay", "127.0.0.1/32", False, 200),
|
||||
("safe-onedomain-nojsreplay", "192.0.2.0/24", False, 302),
|
||||
],
|
||||
)
|
||||
def test_agent_websocket_matches_http_security_policy(agent_server, live_opencode, mode, proxy_whitelist, use_cookie, http_status):
|
||||
from urllib.parse import urlsplit
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import InvalidStatus
|
||||
|
||||
server_url, data_dir, process = agent_server
|
||||
cookie = _login_cookie(server_url)
|
||||
created = requests.post(
|
||||
server_url + "/admin/agent/opencode/pty",
|
||||
headers={"Cookie": cookie, "Origin": server_url},
|
||||
json={"command": "/bin/sh", "args": []},
|
||||
timeout=10,
|
||||
)
|
||||
assert created.status_code == 200
|
||||
pty_id = created.json()["id"]
|
||||
stop_archivebox_process(process)
|
||||
_set_archivebox_config(
|
||||
data_dir,
|
||||
f"SERVER_SECURITY_MODE={mode}",
|
||||
f"REVERSE_PROXY_WHITELIST={proxy_whitelist}",
|
||||
"REVERSE_PROXY_USER_HEADER=Remote-User",
|
||||
)
|
||||
restarted = start_archivebox_server(
|
||||
data_dir,
|
||||
port=urlsplit(server_url).port,
|
||||
env=live_opencode.config.env,
|
||||
log_name="agent-security-server.log",
|
||||
)
|
||||
headers = {"Cookie": cookie} if use_cookie else {"Remote-User": "agent-browser-test"}
|
||||
try:
|
||||
response = requests.get(server_url + "/admin/agent/opencode/global/health", headers=headers, allow_redirects=False, timeout=10)
|
||||
assert response.status_code == http_status
|
||||
|
||||
async def check_socket():
|
||||
connection = connect(
|
||||
server_url.replace("http", "ws", 1) + f"/admin/agent/opencode/pty/{pty_id}/connect",
|
||||
origin=server_url,
|
||||
additional_headers=headers,
|
||||
proxy=None,
|
||||
)
|
||||
if http_status != 200:
|
||||
with pytest.raises(InvalidStatus) as error:
|
||||
async with connection:
|
||||
pytest.fail("Disabled control plane or untrusted proxy accepted a WebSocket")
|
||||
assert error.value.response.status_code == 403
|
||||
return
|
||||
async with connection as socket, asyncio.timeout(10):
|
||||
await socket.send("printf 'ABX_%s\\n' PROXY_OK\n")
|
||||
output = ""
|
||||
while "ABX_PROXY_OK" not in output:
|
||||
chunk = await socket.recv()
|
||||
output += chunk.decode() if isinstance(chunk, bytes) else chunk
|
||||
assert "ABX_PROXY_OK" in output
|
||||
|
||||
asyncio.run(check_socket())
|
||||
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
|
||||
finally:
|
||||
deleted = requests.delete(live_opencode.settings["origin"] + f"/pty/{pty_id}", timeout=10)
|
||||
assert deleted.status_code == 200
|
||||
stop_archivebox_process(restarted)
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Exercise the real agent wrapper with Chromium's native storage failures."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from .test_opencode_agent_1 import installed_opencode as installed_opencode
|
||||
from .test_opencode_agent_1 import live_opencode as live_opencode
|
||||
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
|
||||
from .test_server_security_browser import browser_runtime as browser_runtime
|
||||
|
||||
|
||||
from .test_opencode_browser_1 import (
|
||||
pytestmark as pytestmark,
|
||||
agent_server as agent_server,
|
||||
_login_cookie as _login_cookie,
|
||||
)
|
||||
|
||||
|
||||
def test_agent_websocket_rejects_unauthorized_access(agent_server):
|
||||
from websockets.asyncio.client import connect
|
||||
from websockets.exceptions import InvalidStatus
|
||||
|
||||
server_url, _, _ = agent_server
|
||||
admin_cookie = _login_cookie(server_url)
|
||||
regular_cookie = _login_cookie(server_url, "agent-regular-test")
|
||||
|
||||
async def check_denials():
|
||||
for path, cookie, origin in (
|
||||
("/admin/agent/opencode/pty/unknown/connect", "", server_url),
|
||||
("/admin/agent/opencode/pty/unknown/connect", regular_cookie, server_url),
|
||||
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, "https://untrusted.example"),
|
||||
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, None),
|
||||
("/health/", admin_cookie, server_url),
|
||||
):
|
||||
with pytest.raises(InvalidStatus) as error:
|
||||
async with connect(
|
||||
server_url.replace("http", "ws", 1) + path,
|
||||
origin=origin,
|
||||
additional_headers={"Cookie": cookie},
|
||||
proxy=None,
|
||||
):
|
||||
pytest.fail("Unauthorized WebSocket was accepted")
|
||||
assert error.value.response.status_code == 403
|
||||
|
||||
asyncio.run(check_denials())
|
||||
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
|
||||
|
||||
|
||||
def test_agent_preserves_projects_and_survives_storage_failure(agent_server, browser_runtime):
|
||||
server_url, data_dir, _ = agent_server
|
||||
script = r"""
|
||||
const assert = require('node:assert/strict');
|
||||
const puppeteer = require('puppeteer');
|
||||
const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
|
||||
(async () => {
|
||||
for (const disabled of [false, true]) {
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: config.chrome,
|
||||
headless: true,
|
||||
// Headless interaction must not depend on the desktop display clock.
|
||||
args: ['--no-sandbox', '--disable-frame-rate-limit', ...(disabled ? ['--disable-local-storage'] : [])],
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.goto(new URL('/admin/login/', config.url).href, {waitUntil: 'domcontentloaded'});
|
||||
await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
|
||||
await page.locator('#login-form input[name="password"]').fill('test-password');
|
||||
assert.equal(await page.$eval('#login-form input[name="username"]', element => element.value), 'agent-browser-test');
|
||||
assert.equal(await page.$eval('#login-form input[name="password"]', element => element.value), 'test-password');
|
||||
await Promise.all([
|
||||
page.waitForNavigation({waitUntil: 'domcontentloaded'}),
|
||||
page.locator('#login-form input[type="submit"]').click(),
|
||||
]);
|
||||
assert.ok(!page.url().includes('/admin/login/'), page.url());
|
||||
assert.equal((await page.goto(config.url, {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
const welcome = '#opencode-agent-welcome';
|
||||
await page.waitForSelector(welcome, {visible: true});
|
||||
if (disabled) {
|
||||
assert.equal(await page.evaluate(() => {
|
||||
try { return localStorage === null; }
|
||||
catch (error) { if (error.name !== 'SecurityError') throw error; return true; }
|
||||
}), true);
|
||||
} else {
|
||||
await page.click('#opencode-agent-welcome-dismiss');
|
||||
await page.waitForSelector(welcome, {hidden: true});
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
assert.equal(await page.$eval(welcome, element => element.hidden), true);
|
||||
const existing = {
|
||||
list: [{type: 'http', http: {url: 'http://other.example'}}],
|
||||
projects: {
|
||||
local: [{worktree: '/other-project', expanded: false}],
|
||||
'http://other.example': [{worktree: '/remote-project', expanded: true}],
|
||||
},
|
||||
lastProject: {'http://other.example': '/remote-project'},
|
||||
};
|
||||
await page.evaluate(value => localStorage.setItem('opencode.global.dat:server', JSON.stringify(value)), existing);
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
await page.waitForFunction(workdir => {
|
||||
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
|
||||
return state.projects.local?.some(project => project.worktree === workdir);
|
||||
}, {}, config.workdir);
|
||||
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('opencode.global.dat:server')));
|
||||
assert.deepEqual(saved.list, existing.list);
|
||||
assert.deepEqual(saved.projects['http://other.example'], existing.projects['http://other.example']);
|
||||
assert.equal(saved.lastProject['http://other.example'], '/remote-project');
|
||||
const projects = saved.projects.local;
|
||||
assert.ok(Array.isArray(projects), JSON.stringify({url: page.url(), saved, existing}));
|
||||
assert.deepEqual(projects.find(project => project.worktree === '/other-project'), {worktree: '/other-project', expanded: false});
|
||||
assert.equal(projects.filter(project => project.worktree === config.workdir).length, 1);
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
assert.equal(await page.evaluate(workdir => {
|
||||
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
|
||||
return state.projects.local.filter(project => project.worktree === workdir).length;
|
||||
}, config.workdir), 1);
|
||||
|
||||
// Fill the actual browser quota; do not replace or intercept Storage methods.
|
||||
const failure = await page.evaluate(() => {
|
||||
localStorage.clear();
|
||||
let low = 0, high = 8 * 1024 * 1024;
|
||||
while (low < high) {
|
||||
const size = Math.ceil((low + high) / 2);
|
||||
try { localStorage.setItem('padding', 'x'.repeat(size)); low = size; }
|
||||
catch (error) { if (error.name !== 'QuotaExceededError') throw error; high = size - 1; }
|
||||
}
|
||||
try { localStorage.setItem('extra', '1'); return null; }
|
||||
catch (error) { return error.name; }
|
||||
});
|
||||
assert.equal(failure, 'QuotaExceededError');
|
||||
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
|
||||
await page.waitForSelector(welcome, {visible: true});
|
||||
}
|
||||
await page.click('#opencode-agent-welcome-dismiss');
|
||||
await page.waitForSelector(welcome, {hidden: true});
|
||||
assert.equal((await page.goto(new URL('/add/', config.url).href, {waitUntil: 'domcontentloaded'})).status(), 200);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
console.log('AGENT_STORAGE_FAILURE_ISOLATED');
|
||||
})().catch(error => { console.error(error); process.exitCode = 1; });
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[str(browser_runtime["node_binary"]), "-e", script],
|
||||
input=json.dumps(
|
||||
{
|
||||
"chrome": str(browser_runtime["chrome_binary"]),
|
||||
"url": f"{server_url}/admin/agent",
|
||||
"workdir": str(data_dir.resolve()),
|
||||
},
|
||||
),
|
||||
env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=120,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert "AGENT_STORAGE_FAILURE_ISOLATED" in result.stdout
|
||||
@@ -91,7 +91,12 @@ def test_tlsnotary_preview_verifies_saved_evidence_without_remote_requests(snaps
|
||||
assert "Signed response byte range" in topology.inner_text()
|
||||
assert topology.locator('a[href*="response.http"]').count() > 0
|
||||
assert topology.locator('a[href*="receipt.json"]').count() > 0
|
||||
assert page.locator("a", has_text="Verify independently").get_attribute("href") == "https://tlsnotary.zervice.io/"
|
||||
# The installed plugin owns its verifier domain. Compare its template
|
||||
# with its declared default so a plugin release does not require a
|
||||
# simultaneous core test edit, while stale template links still fail.
|
||||
schema = json.loads((Path(get_plugins_dir()) / "tlsnotary" / "config.json").read_text())
|
||||
verifier_url = schema["properties"]["TLSNOTARY_VERIFIER_URL"]["default"]
|
||||
assert page.locator("a", has_text="Verify independently").get_attribute("href") == f"{verifier_url.rstrip('/')}/"
|
||||
assert all(urlsplit(request).hostname == hostname for request in requests)
|
||||
page.screenshot(path=str(tmp_path / "tlsnotary-desktop.png"), full_page=True)
|
||||
page.set_viewport_size({"width": 390, "height": 844})
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Integration tests for recursive crawling functionality."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def run_add_until(args, env, condition, timeout=120):
|
||||
assert args[0] == "archivebox"
|
||||
result = run_archivebox_cmd(
|
||||
args[1:],
|
||||
cwd=Path.cwd(),
|
||||
env=env,
|
||||
timeout=timeout,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
with use_archivebox_db("."):
|
||||
assert condition(), f"Condition was false after command completed: {' '.join(args)}"
|
||||
return result.stdout, result.stderr
|
||||
|
||||
|
||||
@pytest.mark.timeout(1200)
|
||||
def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
|
||||
initialized_archive,
|
||||
free_tcp_port_factory,
|
||||
recursive_test_site,
|
||||
):
|
||||
"""Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
|
||||
|
||||
from archivebox.plugins.discovery import get_plugin_catalog
|
||||
|
||||
root_url = recursive_test_site["root_url"]
|
||||
plugin_selection = ",".join(sorted(plugin for plugin in get_plugin_catalog() if not plugin.startswith("claude")))
|
||||
env = os.environ.copy()
|
||||
for preinstalled_path_key in (
|
||||
"CHROME_BINARY",
|
||||
"LIB_DIR",
|
||||
"DATA_DIR",
|
||||
"NODE_MODULES_DIR",
|
||||
"NODE_PATH",
|
||||
"PNPM_BIN_DIR",
|
||||
"NPM_BIN_DIR",
|
||||
"CHROMEWEBSTORE_EXTENSIONS_DIR",
|
||||
):
|
||||
env.pop(preinstalled_path_key, None)
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
|
||||
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
|
||||
"CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
|
||||
"TIMEOUT": "90",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
|
||||
"SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
|
||||
"SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
|
||||
"CHROME_HEADLESS": "true",
|
||||
"CHROME_SANDBOX": "false",
|
||||
"CHROME_ISOLATION": "snapshot",
|
||||
"LITEPARSE_OCR_ENABLED": "false",
|
||||
"LITEPARSE_MAX_SOURCES": "4",
|
||||
},
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=2",
|
||||
"--max-urls=8",
|
||||
"--crawl-max-size=100mb",
|
||||
"--tag=recursive-all-plugins",
|
||||
"--parser=url_list",
|
||||
f"--plugins={plugin_selection}",
|
||||
root_url,
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=1200,
|
||||
)
|
||||
stdout, stderr = result.stdout, result.stderr
|
||||
|
||||
if stderr:
|
||||
print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n")
|
||||
if stdout:
|
||||
print(f"\n=== STDOUT (last 4000 chars) ===\n{stdout[-4000:]}\n=== END STDOUT ===\n")
|
||||
assert result.returncode == 0, stderr or stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get(tags_str="recursive-all-plugins")
|
||||
snapshots = list(
|
||||
Snapshot.objects.filter(crawl=crawl)
|
||||
.order_by("depth", "url")
|
||||
.values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"),
|
||||
)
|
||||
archive_results = list(
|
||||
ArchiveResult.objects.filter(snapshot__crawl=crawl)
|
||||
.select_related("snapshot")
|
||||
.order_by("snapshot__depth", "snapshot__url", "plugin", "hook_name")
|
||||
.values_list(
|
||||
"snapshot_id",
|
||||
"snapshot__url",
|
||||
"snapshot__depth",
|
||||
"plugin",
|
||||
"hook_name",
|
||||
"status",
|
||||
"output_files",
|
||||
"output_size",
|
||||
"output_str",
|
||||
),
|
||||
)
|
||||
process_snapshot_ids = {
|
||||
process_id: str(snapshot_id)
|
||||
for snapshot_id, process_id in ArchiveResult.objects.filter(
|
||||
snapshot__crawl=crawl,
|
||||
process_id__isnull=False,
|
||||
).values_list("snapshot_id", "process_id")
|
||||
}
|
||||
processes = list(
|
||||
Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids)
|
||||
.order_by("started_at")
|
||||
.values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
|
||||
)
|
||||
|
||||
assert crawl.max_depth == 2
|
||||
assert crawl.config["CRAWL_MAX_URLS"] == 8
|
||||
assert crawl.config["CRAWL_MAX_SIZE"] == 100 * 1024 * 1024
|
||||
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
|
||||
assert crawl.status == Crawl.StatusChoices.SEALED
|
||||
assert crawl.retry_at is None
|
||||
|
||||
expected_urls = {
|
||||
root_url,
|
||||
*recursive_test_site["child_urls"],
|
||||
*recursive_test_site["deep_urls"],
|
||||
}
|
||||
snapshots_by_url = {url: (snapshot_id, depth, parent_id) for snapshot_id, url, depth, _status, parent_id, _downloaded_at in snapshots}
|
||||
assert set(snapshots_by_url) == expected_urls
|
||||
root_id, root_depth, root_parent_id = snapshots_by_url[root_url]
|
||||
assert root_depth == 0
|
||||
assert root_parent_id is None
|
||||
child_ids_by_url = {}
|
||||
for child_url in recursive_test_site["child_urls"]:
|
||||
child_id, child_depth, child_parent_id = snapshots_by_url[child_url]
|
||||
assert child_depth == 1
|
||||
assert child_parent_id == root_id
|
||||
child_ids_by_url[child_url] = child_id
|
||||
for child_url, deep_url in zip(recursive_test_site["child_urls"], recursive_test_site["deep_urls"], strict=True):
|
||||
_deep_id, deep_depth, deep_parent_id = snapshots_by_url[deep_url]
|
||||
assert deep_depth == 2
|
||||
assert deep_parent_id == child_ids_by_url[child_url]
|
||||
assert all(status == Snapshot.StatusChoices.SEALED for _id, _url, _depth, status, _parent, _downloaded_at in snapshots)
|
||||
assert all(downloaded_at is not None for _id, _url, _depth, _status, _parent, downloaded_at in snapshots)
|
||||
|
||||
assert archive_results
|
||||
allowed_statuses = {
|
||||
ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
ArchiveResult.StatusChoices.NORESULTS,
|
||||
ArchiveResult.StatusChoices.SKIPPED,
|
||||
}
|
||||
unexpected_results = [
|
||||
{
|
||||
"url": url,
|
||||
"depth": depth,
|
||||
"plugin": plugin,
|
||||
"hook_name": hook_name,
|
||||
"status": status,
|
||||
"output_str": output_str,
|
||||
}
|
||||
for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
|
||||
if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
|
||||
]
|
||||
# This fixture serves HTTP. TLSNotary must reject every plaintext document;
|
||||
# a setup, timeout, or verifier error must not count as that expected refusal.
|
||||
tlsnotary_results = [row for row in archive_results if row[3] == "tlsnotary"]
|
||||
assert {row[1] for row in tlsnotary_results} == expected_urls
|
||||
assert len(tlsnotary_results) == len(expected_urls)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for snapshot_id, _url, _depth, _plugin, hook_name, status, files, size, _output in tlsnotary_results:
|
||||
assert status == ArchiveResult.StatusChoices.FAILED
|
||||
assert not files
|
||||
assert size == 0
|
||||
process = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="tlsnotary", hook_name=hook_name).process
|
||||
assert process is not None
|
||||
assert process.status == Process.StatusChoices.EXITED
|
||||
assert process.exit_code == 1
|
||||
assert process.stderr.strip() == "[tlsnotary] TLSNotary requires an HTTPS document"
|
||||
assert not [result for result in unexpected_results if result["plugin"] != "tlsnotary"]
|
||||
ytdlp_results = [
|
||||
(url, status, output_str)
|
||||
for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
|
||||
if plugin == "ytdlp"
|
||||
]
|
||||
assert ytdlp_results
|
||||
assert all(status != ArchiveResult.StatusChoices.FAILED for _url, status, _output_str in ytdlp_results), ytdlp_results
|
||||
|
||||
plugins_seen = {plugin for _snapshot_id, _url, _depth, plugin, _hook_name, _status, _files, _size, _output in archive_results}
|
||||
assert {
|
||||
"wget",
|
||||
"headers",
|
||||
"title",
|
||||
"pdf",
|
||||
"screenshot",
|
||||
"dom",
|
||||
"singlefile",
|
||||
"readability",
|
||||
"mercury",
|
||||
"htmltotext",
|
||||
"favicon",
|
||||
"parse_html_urls",
|
||||
"archivedotorg",
|
||||
}.issubset(plugins_seen)
|
||||
|
||||
snapshot_root = initialized_archive / "archive/users/system/snapshots"
|
||||
assert list(snapshot_root.rglob("wget/**/*.html"))
|
||||
assert list(snapshot_root.rglob("headers/**/headers.json"))
|
||||
assert list(snapshot_root.rglob("title/title.txt"))
|
||||
assert list(snapshot_root.rglob("pdf/**/*.pdf"))
|
||||
assert list(snapshot_root.rglob("screenshot/**/*.png"))
|
||||
assert list(snapshot_root.rglob("dom/**/*.html"))
|
||||
assert list(snapshot_root.rglob("singlefile/**/*.html"))
|
||||
assert list(snapshot_root.rglob("readability/**/*.html"))
|
||||
assert list(snapshot_root.rglob("mercury/**/*.html"))
|
||||
assert list(snapshot_root.rglob("htmltotext/**/*.txt"))
|
||||
assert list(snapshot_root.rglob("favicon/**/*"))
|
||||
urls_jsonl_files = list(snapshot_root.rglob("parse_html_urls/urls.jsonl"))
|
||||
assert urls_jsonl_files
|
||||
parsed_urls = {
|
||||
json.loads(line)["url"] for path in urls_jsonl_files for line in path.read_text(errors="ignore").splitlines() if line.strip()
|
||||
}
|
||||
assert set(recursive_test_site["child_urls"]).issubset(parsed_urls)
|
||||
assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
|
||||
|
||||
assert processes
|
||||
assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
|
||||
|
||||
intervals = []
|
||||
for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
|
||||
if not started_at or not ended_at:
|
||||
continue
|
||||
process_snapshot_id = process_snapshot_ids.get(process_id)
|
||||
if process_snapshot_id is None:
|
||||
continue
|
||||
intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))
|
||||
|
||||
overlapping = [
|
||||
(left, right)
|
||||
for index, left in enumerate(intervals)
|
||||
for right in intervals[index + 1 :]
|
||||
if left[0] != right[0] and left[1] < right[2] and right[1] < left[2]
|
||||
]
|
||||
assert overlapping, f"Expected hook processes from different snapshots to overlap, got intervals: {intervals}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Integration tests for recursive crawling functionality."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import run_archivebox_cmd
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
from .test_recursive_crawl_1 import (
|
||||
pytestmark as pytestmark,
|
||||
run_add_until as run_add_until,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(600)
|
||||
def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(initialized_archive, recursive_test_site):
|
||||
"""A real three-URL capture must not launch all Chrome hook trees at once."""
|
||||
urls = [recursive_test_site["root_url"], *recursive_test_site["child_urls"][:2]]
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
|
||||
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
|
||||
"CHROME_HEADLESS": "true",
|
||||
"CHROME_SANDBOX": "false",
|
||||
"CHROME_ISOLATION": "crawl",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
|
||||
"TIMEOUT": "90",
|
||||
},
|
||||
)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--parser=url_list", "--plugins=chrome,wget,screenshot", "--tag=measured-admission", *urls],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=600,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get(tags_str="measured-admission")
|
||||
snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("created_at"))
|
||||
hook_intervals = list(
|
||||
Process.objects.filter(
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
archiveresult__snapshot__crawl=crawl,
|
||||
)
|
||||
.values_list("archiveresult__snapshot_id", "started_at", "ended_at")
|
||||
.distinct(),
|
||||
)
|
||||
|
||||
assert len(snapshots) == 3
|
||||
assert all(snapshot.status == Snapshot.StatusChoices.SEALED and snapshot.downloaded_at for snapshot in snapshots)
|
||||
first_id = snapshots[0].id
|
||||
first_end = snapshots[0].downloaded_at
|
||||
later_starts = [started for snapshot_id, started, _ended in hook_intervals if snapshot_id != first_id and started]
|
||||
assert later_starts and first_end <= min(later_starts), hook_intervals
|
||||
later_intervals = [
|
||||
(snapshot_id, started, ended) for snapshot_id, started, ended in hook_intervals if snapshot_id != first_id and started and ended
|
||||
]
|
||||
assert any(
|
||||
left_id != right_id and left_start < right_end and right_start < left_end
|
||||
for index, (left_id, left_start, left_end) in enumerate(later_intervals)
|
||||
for right_id, right_start, right_end in later_intervals[index + 1 :]
|
||||
), later_intervals
|
||||
|
||||
|
||||
def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, initialized_archive):
|
||||
"""Test that Snapshot model has parent_snapshot field."""
|
||||
|
||||
column_names = {field.column for field in Snapshot._meta.local_fields}
|
||||
|
||||
assert "parent_snapshot_id" in column_names, f"Snapshot table should have parent_snapshot_id column. Columns: {column_names}"
|
||||
+4
-313
@@ -13,21 +13,10 @@ from archivebox.machine.models import Binary, Process
|
||||
from archivebox.tests.conftest import run_archivebox_cmd, cli_env
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def run_add_until(args, env, condition, timeout=120):
|
||||
assert args[0] == "archivebox"
|
||||
result = run_archivebox_cmd(
|
||||
args[1:],
|
||||
cwd=Path.cwd(),
|
||||
env=env,
|
||||
timeout=timeout,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
with use_archivebox_db("."):
|
||||
assert condition(), f"Condition was false after command completed: {' '.join(args)}"
|
||||
return result.stdout, result.stderr
|
||||
from .test_recursive_crawl_1 import (
|
||||
pytestmark as pytestmark,
|
||||
run_add_until as run_add_until,
|
||||
)
|
||||
|
||||
|
||||
def test_background_hooks_dont_block_parser_extractors(tmp_path, initialized_archive, recursive_test_site):
|
||||
@@ -492,300 +481,6 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(1200)
|
||||
def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
|
||||
initialized_archive,
|
||||
free_tcp_port_factory,
|
||||
recursive_test_site,
|
||||
):
|
||||
"""Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
|
||||
|
||||
from archivebox.plugins.discovery import get_plugin_catalog
|
||||
|
||||
root_url = recursive_test_site["root_url"]
|
||||
plugin_selection = ",".join(sorted(plugin for plugin in get_plugin_catalog() if not plugin.startswith("claude")))
|
||||
env = os.environ.copy()
|
||||
for preinstalled_path_key in (
|
||||
"CHROME_BINARY",
|
||||
"LIB_DIR",
|
||||
"DATA_DIR",
|
||||
"NODE_MODULES_DIR",
|
||||
"NODE_PATH",
|
||||
"PNPM_BIN_DIR",
|
||||
"NPM_BIN_DIR",
|
||||
"CHROMEWEBSTORE_EXTENSIONS_DIR",
|
||||
):
|
||||
env.pop(preinstalled_path_key, None)
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
|
||||
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
|
||||
"CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
|
||||
"TIMEOUT": "90",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
|
||||
"SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
|
||||
"SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
|
||||
"CHROME_HEADLESS": "true",
|
||||
"CHROME_SANDBOX": "false",
|
||||
"CHROME_ISOLATION": "snapshot",
|
||||
"LITEPARSE_OCR_ENABLED": "false",
|
||||
"LITEPARSE_MAX_SOURCES": "4",
|
||||
},
|
||||
)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=2",
|
||||
"--max-urls=8",
|
||||
"--crawl-max-size=100mb",
|
||||
"--tag=recursive-all-plugins",
|
||||
"--parser=url_list",
|
||||
f"--plugins={plugin_selection}",
|
||||
root_url,
|
||||
],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=1200,
|
||||
)
|
||||
stdout, stderr = result.stdout, result.stderr
|
||||
|
||||
if stderr:
|
||||
print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n")
|
||||
if stdout:
|
||||
print(f"\n=== STDOUT (last 4000 chars) ===\n{stdout[-4000:]}\n=== END STDOUT ===\n")
|
||||
assert result.returncode == 0, stderr or stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get(tags_str="recursive-all-plugins")
|
||||
snapshots = list(
|
||||
Snapshot.objects.filter(crawl=crawl)
|
||||
.order_by("depth", "url")
|
||||
.values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"),
|
||||
)
|
||||
archive_results = list(
|
||||
ArchiveResult.objects.filter(snapshot__crawl=crawl)
|
||||
.select_related("snapshot")
|
||||
.order_by("snapshot__depth", "snapshot__url", "plugin", "hook_name")
|
||||
.values_list(
|
||||
"snapshot_id",
|
||||
"snapshot__url",
|
||||
"snapshot__depth",
|
||||
"plugin",
|
||||
"hook_name",
|
||||
"status",
|
||||
"output_files",
|
||||
"output_size",
|
||||
"output_str",
|
||||
),
|
||||
)
|
||||
process_snapshot_ids = {
|
||||
process_id: str(snapshot_id)
|
||||
for snapshot_id, process_id in ArchiveResult.objects.filter(
|
||||
snapshot__crawl=crawl,
|
||||
process_id__isnull=False,
|
||||
).values_list("snapshot_id", "process_id")
|
||||
}
|
||||
processes = list(
|
||||
Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids)
|
||||
.order_by("started_at")
|
||||
.values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
|
||||
)
|
||||
|
||||
assert crawl.max_depth == 2
|
||||
assert crawl.config["CRAWL_MAX_URLS"] == 8
|
||||
assert crawl.config["CRAWL_MAX_SIZE"] == 100 * 1024 * 1024
|
||||
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
|
||||
assert crawl.status == Crawl.StatusChoices.SEALED
|
||||
assert crawl.retry_at is None
|
||||
|
||||
expected_urls = {
|
||||
root_url,
|
||||
*recursive_test_site["child_urls"],
|
||||
*recursive_test_site["deep_urls"],
|
||||
}
|
||||
snapshots_by_url = {url: (snapshot_id, depth, parent_id) for snapshot_id, url, depth, _status, parent_id, _downloaded_at in snapshots}
|
||||
assert set(snapshots_by_url) == expected_urls
|
||||
root_id, root_depth, root_parent_id = snapshots_by_url[root_url]
|
||||
assert root_depth == 0
|
||||
assert root_parent_id is None
|
||||
child_ids_by_url = {}
|
||||
for child_url in recursive_test_site["child_urls"]:
|
||||
child_id, child_depth, child_parent_id = snapshots_by_url[child_url]
|
||||
assert child_depth == 1
|
||||
assert child_parent_id == root_id
|
||||
child_ids_by_url[child_url] = child_id
|
||||
for child_url, deep_url in zip(recursive_test_site["child_urls"], recursive_test_site["deep_urls"], strict=True):
|
||||
_deep_id, deep_depth, deep_parent_id = snapshots_by_url[deep_url]
|
||||
assert deep_depth == 2
|
||||
assert deep_parent_id == child_ids_by_url[child_url]
|
||||
assert all(status == Snapshot.StatusChoices.SEALED for _id, _url, _depth, status, _parent, _downloaded_at in snapshots)
|
||||
assert all(downloaded_at is not None for _id, _url, _depth, _status, _parent, downloaded_at in snapshots)
|
||||
|
||||
assert archive_results
|
||||
allowed_statuses = {
|
||||
ArchiveResult.StatusChoices.SUCCEEDED,
|
||||
ArchiveResult.StatusChoices.NORESULTS,
|
||||
ArchiveResult.StatusChoices.SKIPPED,
|
||||
}
|
||||
unexpected_results = [
|
||||
{
|
||||
"url": url,
|
||||
"depth": depth,
|
||||
"plugin": plugin,
|
||||
"hook_name": hook_name,
|
||||
"status": status,
|
||||
"output_str": output_str,
|
||||
}
|
||||
for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
|
||||
if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
|
||||
]
|
||||
# This fixture serves HTTP. TLSNotary must reject every plaintext document;
|
||||
# a setup, timeout, or verifier error must not count as that expected refusal.
|
||||
tlsnotary_results = [row for row in archive_results if row[3] == "tlsnotary"]
|
||||
assert {row[1] for row in tlsnotary_results} == expected_urls
|
||||
assert len(tlsnotary_results) == len(expected_urls)
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for snapshot_id, _url, _depth, _plugin, hook_name, status, files, size, _output in tlsnotary_results:
|
||||
assert status == ArchiveResult.StatusChoices.FAILED
|
||||
assert not files
|
||||
assert size == 0
|
||||
process = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="tlsnotary", hook_name=hook_name).process
|
||||
assert process is not None
|
||||
assert process.status == Process.StatusChoices.EXITED
|
||||
assert process.exit_code == 1
|
||||
assert process.stderr.strip() == "[tlsnotary] TLSNotary requires an HTTPS document"
|
||||
assert not [result for result in unexpected_results if result["plugin"] != "tlsnotary"]
|
||||
ytdlp_results = [
|
||||
(url, status, output_str)
|
||||
for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
|
||||
if plugin == "ytdlp"
|
||||
]
|
||||
assert ytdlp_results
|
||||
assert all(status != ArchiveResult.StatusChoices.FAILED for _url, status, _output_str in ytdlp_results), ytdlp_results
|
||||
|
||||
plugins_seen = {plugin for _snapshot_id, _url, _depth, plugin, _hook_name, _status, _files, _size, _output in archive_results}
|
||||
assert {
|
||||
"wget",
|
||||
"headers",
|
||||
"title",
|
||||
"pdf",
|
||||
"screenshot",
|
||||
"dom",
|
||||
"singlefile",
|
||||
"readability",
|
||||
"mercury",
|
||||
"htmltotext",
|
||||
"favicon",
|
||||
"parse_html_urls",
|
||||
"archivedotorg",
|
||||
}.issubset(plugins_seen)
|
||||
|
||||
snapshot_root = initialized_archive / "archive/users/system/snapshots"
|
||||
assert list(snapshot_root.rglob("wget/**/*.html"))
|
||||
assert list(snapshot_root.rglob("headers/**/headers.json"))
|
||||
assert list(snapshot_root.rglob("title/title.txt"))
|
||||
assert list(snapshot_root.rglob("pdf/**/*.pdf"))
|
||||
assert list(snapshot_root.rglob("screenshot/**/*.png"))
|
||||
assert list(snapshot_root.rglob("dom/**/*.html"))
|
||||
assert list(snapshot_root.rglob("singlefile/**/*.html"))
|
||||
assert list(snapshot_root.rglob("readability/**/*.html"))
|
||||
assert list(snapshot_root.rglob("mercury/**/*.html"))
|
||||
assert list(snapshot_root.rglob("htmltotext/**/*.txt"))
|
||||
assert list(snapshot_root.rglob("favicon/**/*"))
|
||||
urls_jsonl_files = list(snapshot_root.rglob("parse_html_urls/urls.jsonl"))
|
||||
assert urls_jsonl_files
|
||||
parsed_urls = {
|
||||
json.loads(line)["url"] for path in urls_jsonl_files for line in path.read_text(errors="ignore").splitlines() if line.strip()
|
||||
}
|
||||
assert set(recursive_test_site["child_urls"]).issubset(parsed_urls)
|
||||
assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
|
||||
|
||||
assert processes
|
||||
assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
|
||||
|
||||
intervals = []
|
||||
for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
|
||||
if not started_at or not ended_at:
|
||||
continue
|
||||
process_snapshot_id = process_snapshot_ids.get(process_id)
|
||||
if process_snapshot_id is None:
|
||||
continue
|
||||
intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))
|
||||
|
||||
overlapping = [
|
||||
(left, right)
|
||||
for index, left in enumerate(intervals)
|
||||
for right in intervals[index + 1 :]
|
||||
if left[0] != right[0] and left[1] < right[2] and right[1] < left[2]
|
||||
]
|
||||
assert overlapping, f"Expected hook processes from different snapshots to overlap, got intervals: {intervals}"
|
||||
|
||||
|
||||
@pytest.mark.timeout(600)
|
||||
def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(initialized_archive, recursive_test_site):
|
||||
"""A real three-URL capture must not launch all Chrome hook trees at once."""
|
||||
urls = [recursive_test_site["root_url"], *recursive_test_site["child_urls"][:2]]
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"USE_COLOR": "false",
|
||||
"SHOW_PROGRESS": "false",
|
||||
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
|
||||
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
|
||||
"CHROME_HEADLESS": "true",
|
||||
"CHROME_SANDBOX": "false",
|
||||
"CHROME_ISOLATION": "crawl",
|
||||
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
|
||||
"TIMEOUT": "90",
|
||||
},
|
||||
)
|
||||
result = run_archivebox_cmd(
|
||||
["add", "--parser=url_list", "--plugins=chrome,wget,screenshot", "--tag=measured-admission", *urls],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=600,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
with use_archivebox_db(initialized_archive):
|
||||
crawl = Crawl.objects.get(tags_str="measured-admission")
|
||||
snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("created_at"))
|
||||
hook_intervals = list(
|
||||
Process.objects.filter(
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
archiveresult__snapshot__crawl=crawl,
|
||||
)
|
||||
.values_list("archiveresult__snapshot_id", "started_at", "ended_at")
|
||||
.distinct(),
|
||||
)
|
||||
|
||||
assert len(snapshots) == 3
|
||||
assert all(snapshot.status == Snapshot.StatusChoices.SEALED and snapshot.downloaded_at for snapshot in snapshots)
|
||||
first_id = snapshots[0].id
|
||||
first_end = snapshots[0].downloaded_at
|
||||
later_starts = [started for snapshot_id, started, _ended in hook_intervals if snapshot_id != first_id and started]
|
||||
assert later_starts and first_end <= min(later_starts), hook_intervals
|
||||
later_intervals = [
|
||||
(snapshot_id, started, ended) for snapshot_id, started, ended in hook_intervals if snapshot_id != first_id and started and ended
|
||||
]
|
||||
assert any(
|
||||
left_id != right_id and left_start < right_end and right_start < left_end
|
||||
for index, (left_id, left_start, left_end) in enumerate(later_intervals)
|
||||
for right_id, right_start, right_end in later_intervals[index + 1 :]
|
||||
), later_intervals
|
||||
|
||||
|
||||
def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, initialized_archive):
|
||||
"""Test that Snapshot model has parent_snapshot field."""
|
||||
|
||||
column_names = {field.column for field in Snapshot._meta.local_fields}
|
||||
|
||||
assert "parent_snapshot_id" in column_names, f"Snapshot table should have parent_snapshot_id column. Columns: {column_names}"
|
||||
|
||||
|
||||
def test_snapshot_depth_field_exists(tmp_path, initialized_archive):
|
||||
"""Test that Snapshot model has depth field."""
|
||||
|
||||
@@ -857,7 +552,3 @@ def test_archiveresult_worker_queue_filters_by_foreground_extractors(tmp_path, i
|
||||
assert len(non_queued) > 0 or len(parser_status) == 0, (
|
||||
f"With {len(bg_results)} background hooks started, parser extractors should still run. Got statuses: {parser_statuses}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -14,7 +14,7 @@ import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
RELEASE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release.yml"
|
||||
RELEASE_CANDIDATE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release-candidate.yml"
|
||||
CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml"
|
||||
PIP_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pip.yml"
|
||||
DOCKER_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "docker.yml"
|
||||
VERIFY_STAGING_SCRIPT = REPO_ROOT / "bin" / "verify-staging-release.py"
|
||||
@@ -115,9 +115,9 @@ def test_stable_release_reconciles_concurrent_version_only_dev_bump(tmp_path, de
|
||||
assert result.returncode == 0, result.stderr
|
||||
merged = _git(work, "ls-remote", "origin", "refs/heads/dev").split()[0]
|
||||
|
||||
channel_step = next(
|
||||
step for step in yaml.safe_load(RELEASE_CANDIDATE_WORKFLOW.read_text())["jobs"]["prepare"]["steps"] if step.get("id") == "channel"
|
||||
)
|
||||
prepare_path = yaml.safe_load(CI_WORKFLOW.read_text())["jobs"]["prepare"]["uses"]
|
||||
prepare = yaml.safe_load((REPO_ROOT / prepare_path).read_text())
|
||||
channel_step = next(step for step in prepare["jobs"]["prepare"]["steps"] if step.get("id") == "channel")
|
||||
|
||||
def stable_channel(sha, branch="dev"):
|
||||
output = tmp_path / "channel-output"
|
||||
@@ -140,7 +140,7 @@ def test_stable_release_reconciles_concurrent_version_only_dev_bump(tmp_path, de
|
||||
assert _git(work, "rev-parse", f"{merged}^{{tree}}") == _git(work, "rev-parse", f"{stable}^{{tree}}")
|
||||
assert _git(work, "merge-base", stable, candidate) == base
|
||||
# Reconciliation creates a new merge commit with the tested main tree. Its
|
||||
# candidate workflow must not immediately bump dev and undo synchronization.
|
||||
# version preparation must not immediately bump dev and undo synchronization.
|
||||
assert stable_channel(merged) == "stable=true"
|
||||
assert stable_channel(merged, branch="main") == "stable=false"
|
||||
|
||||
@@ -184,8 +184,28 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
|
||||
assert jobs["cascade"]["if"] == "needs.python-release.outputs.release_ready == 'true'"
|
||||
|
||||
assert all(step.get("name") != "Verify published PyPI package installs and runs" for step in python_release["steps"])
|
||||
candidate = yaml.safe_load(RELEASE_CANDIDATE_WORKFLOW.read_text())
|
||||
assert candidate["jobs"]["python-artifacts"]["with"]["full_tests"] is False
|
||||
ci = yaml.safe_load(CI_WORKFLOW.read_text())
|
||||
screenshots = yaml.safe_load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text())
|
||||
prepare_path = ci["jobs"]["prepare"]["uses"]
|
||||
assert screenshots["jobs"]["prepare"]["uses"] == prepare_path
|
||||
prepare = yaml.safe_load((REPO_ROOT / prepare_path).read_text())
|
||||
assert prepare["jobs"]["prepare"]["steps"][-1]["uses"] == "ArchiveBox/monorepo/.github/actions/prepare-release-version@main"
|
||||
assert screenshots["jobs"]["deploy"]["needs"] == "prepare"
|
||||
assert screenshots["jobs"]["deploy"]["if"] == "needs.prepare.outputs.run_jobs == 'true'"
|
||||
for name, job in ci["jobs"].items():
|
||||
if name not in {"prepare", "required"}:
|
||||
assert job["needs"] == "prepare"
|
||||
assert job["if"] == "needs.prepare.outputs.run_jobs == 'true'"
|
||||
gate = next(step["run"] for step in jobs["candidate"]["steps"] if step.get("id") == "verified")
|
||||
assert 'ARTIFACT_RUN_ID="$CI_RUN_ID"' in gate
|
||||
assert 'DIGEST_RUN_ID="$CI_RUN_ID"' in gate
|
||||
assert 'ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"' not in gate
|
||||
assert 'DIGEST_RUN_ID="$CANDIDATE_RUN_ID"' not in gate
|
||||
docker_workflow = yaml.safe_load(DOCKER_WORKFLOW.read_text())
|
||||
version_check = next(
|
||||
step for step in docker_workflow["jobs"]["build"]["steps"] if step.get("name") == "Validate exact built image version and commit"
|
||||
)
|
||||
assert version_check["if"] == "inputs.full_tests"
|
||||
pip_workflow = yaml.safe_load(PIP_WORKFLOW.read_text())
|
||||
install_script = next(
|
||||
step["run"] for step in pip_workflow["jobs"]["build"]["steps"] if step.get("name") == "Release wheel import and CLI smoke"
|
||||
@@ -415,3 +435,75 @@ def test_staging_acceptance_limits_only_exact_known_auth_and_tls_failures():
|
||||
assert "missing or truncated" in embedded.group(1)
|
||||
assert "output.stat().st_size != metadata.get('size')" in embedded.group(1)
|
||||
assert "tlsnotary reported unexpected status" in embedded.group(1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested", ["", "archivebox/tests/test_urls.py", "archivebox/tests/test_release_workflow.py"])
|
||||
def test_ci_batches_preserve_inventory_runner_ownership_and_single_file_runs(tmp_path, requested):
|
||||
hosted_paths = {
|
||||
path.relative_to(REPO_ROOT).as_posix()
|
||||
for path in (REPO_ROOT / "archivebox/tests").glob("test_*.py")
|
||||
if "# ci-runner: hosted" in path.read_text().splitlines()[:5]
|
||||
}
|
||||
result = subprocess.run(
|
||||
["uv", "run", "--no-cache", "--no-project", "python", str(REPO_ROOT / ".github/scripts/discover_test_matrix.py")],
|
||||
cwd=tmp_path,
|
||||
env={**os.environ, "TEST_FILE": requested, "UGNAS_CI_MAX_JOBS": "3"},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
matrix = json.loads(result.stdout.splitlines()[1])
|
||||
paths = [path for item in matrix for path in item["paths"]]
|
||||
expected = sorted(path.relative_to(REPO_ROOT).as_posix() for path in (REPO_ROOT / "archivebox/tests").glob("test_*.py"))
|
||||
assert sorted(paths) == expected
|
||||
assert len(paths) == len(set(paths))
|
||||
assert len(matrix) < len(paths)
|
||||
assert all(1 <= len(item["paths"]) <= 8 for item in matrix)
|
||||
assert sum(item["ugnas"] for item in matrix) == 3
|
||||
assert not any(path in hosted_paths for item in matrix if item["ugnas"] for path in item["paths"])
|
||||
ldap = next(item for item in matrix if item["extra"] == "ldap")
|
||||
assert ldap["paths"] == ["archivebox/tests/test_auth_ldap.py"]
|
||||
if requested:
|
||||
assert len([item for item in matrix if item["paths"] == [requested]]) == 1
|
||||
|
||||
|
||||
def test_ci_discovery_handles_added_renamed_and_deleted_files_without_timing_edits(tmp_path):
|
||||
script = tmp_path / ".github/scripts/discover_test_matrix.py"
|
||||
script.parent.mkdir(parents=True)
|
||||
shutil.copy2(REPO_ROOT / ".github/scripts/discover_test_matrix.py", script)
|
||||
shutil.copy2(REPO_ROOT / ".github/test-durations.json", tmp_path / ".github/test-durations.json")
|
||||
tests = tmp_path / "archivebox/tests"
|
||||
tests.mkdir(parents=True)
|
||||
for name in ("test_core_config.py", "test_util.py"):
|
||||
shutil.copy2(REPO_ROOT / "archivebox/tests" / name, tests / name)
|
||||
|
||||
def discover():
|
||||
result = subprocess.run(
|
||||
["uv", "run", "--no-cache", "--no-project", "python", str(script)],
|
||||
cwd=tmp_path,
|
||||
env={**os.environ, "TEST_FILE": "", "UGNAS_CI_MAX_JOBS": "3"},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
matrix = json.loads(result.stdout.splitlines()[1])
|
||||
paths = [path for item in matrix for path in item["paths"]]
|
||||
assert sorted(paths) == sorted(path.relative_to(tmp_path).as_posix() for path in tests.glob("test_*.py"))
|
||||
assert len(paths) == len(set(paths))
|
||||
return matrix
|
||||
|
||||
assert len(discover()) == 1
|
||||
(tests / "test_util.py").rename(tests / "test_renamed.py")
|
||||
assert len(discover()) == 2
|
||||
shutil.copy2(REPO_ROOT / "archivebox/tests/test_util.py", tests / "test_added.py")
|
||||
assert len(discover()) == 3
|
||||
(tests / "test_core_config.py").unlink()
|
||||
assert len(discover()) == 2
|
||||
|
||||
source = REPO_ROOT / "archivebox/tests/test_util.py"
|
||||
(tests / "test_runner.py").write_text("# ci-runner: hosted\n" + source.read_text())
|
||||
assert next(item for item in discover() if "archivebox/tests/test_runner.py" in item["paths"])["ugnas"] is False
|
||||
(tests / "test_runner.py").rename(tests / "test_runner_renamed.py")
|
||||
assert next(item for item in discover() if "archivebox/tests/test_runner_renamed.py" in item["paths"])["ugnas"] is False
|
||||
(tests / "test_runner_renamed.py").unlink()
|
||||
assert all(item["ugnas"] for item in discover())
|
||||
@@ -594,7 +594,11 @@ class TestPublicIndexSearch:
|
||||
assert b"Public Example Website" in response.content
|
||||
assert b"No snapshots found." not in response.content
|
||||
|
||||
def test_public_index_shows_exact_total_count_and_page_count_for_100_plus_snapshots(self, client, crawl, public_snapshot):
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_public_index_shows_cached_total_count_and_page_count_for_100_plus_snapshots(self, client, crawl, public_snapshot):
|
||||
import time
|
||||
|
||||
from django.core.cache import cache
|
||||
from archivebox.core.models import Snapshot
|
||||
|
||||
base_time = public_snapshot.bookmarked_at + timedelta(seconds=1)
|
||||
@@ -628,11 +632,21 @@ class TestPublicIndexSearch:
|
||||
],
|
||||
)
|
||||
|
||||
key = "progress-collection:public"
|
||||
cache.delete_many([key, f"{key}:refresh"])
|
||||
initial_response = client.get("/public/", HTTP_HOST=WEB_HOST)
|
||||
assert initial_response.status_code == 200
|
||||
assert initial_response.context["public_snapshot_count"] is None
|
||||
assert b"last »" not in initial_response.content
|
||||
deadline = time.monotonic() + 5
|
||||
while cache.get(key) is None and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert cache.get(key)["snapshots"] == 125
|
||||
response = client.get("/public/", HTTP_HOST=WEB_HOST)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.context["paginator"].count == 125
|
||||
assert response.context["paginator"].num_pages == 3
|
||||
assert response.context["public_snapshot_count"] == 125
|
||||
assert response.context["public_page_count"] == 3
|
||||
assert response.context["page_obj"].has_next() is True
|
||||
assert len(response.context["object_list"]) == 50
|
||||
content = response.content.decode()
|
||||
@@ -1329,7 +1343,7 @@ class TestSearchBackendsE2E:
|
||||
surface_name,
|
||||
search_mode,
|
||||
first_positive_elapsed,
|
||||
counts[:10],
|
||||
count_events[:10],
|
||||
)
|
||||
assert total_elapsed < 2.0, (surface_name, search_mode, total_elapsed, counts[-10:])
|
||||
|
||||
|
||||
@@ -954,6 +954,17 @@ def test_server_security_modes_in_chrome(
|
||||
def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archive: Path, browser_runtime, tmp_path: Path) -> None:
|
||||
from archivebox.core.routes_util import get_snapshot_subdomain
|
||||
|
||||
node_binary = Path(browser_runtime["node_binary"])
|
||||
node_target = node_binary.resolve(strict=True)
|
||||
node_version = subprocess.check_output([str(node_binary), "--version"], text=True).strip()
|
||||
|
||||
def assert_browser_runtime_unchanged(stage):
|
||||
actual_target = node_binary.resolve(strict=True)
|
||||
actual_version = subprocess.check_output([str(node_binary), "--version"], text=True).strip()
|
||||
assert (actual_target, actual_version) == (node_target, node_version), (
|
||||
f"{stage} changed the browser runtime: {node_target} ({node_version}) -> {actual_target} ({actual_version})"
|
||||
)
|
||||
|
||||
url = "https://example.com"
|
||||
port = get_free_port()
|
||||
env = cli_env(
|
||||
@@ -992,6 +1003,7 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
|
||||
assert install_result.returncode == 0, (
|
||||
f"archivebox install archivewebpage failed:\nSTDOUT:\n{install_result.stdout}\nSTDERR:\n{install_result.stderr}"
|
||||
)
|
||||
assert_browser_runtime_unchanged("archivebox install archivewebpage")
|
||||
|
||||
process = start_daemon_server(
|
||||
initialized_archive,
|
||||
@@ -1001,6 +1013,7 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
|
||||
log_name="archivewebpage_server.log",
|
||||
)
|
||||
get_http_response(port, host=f"archivebox.localhost:{port}", path="/")
|
||||
assert_browser_runtime_unchanged("archivebox server startup")
|
||||
command_result = run_archivebox_cmd(
|
||||
["add", "--depth=0", "--max-urls=1", "--plugins=archivewebpage", url],
|
||||
cwd=initialized_archive,
|
||||
@@ -1008,6 +1021,7 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
|
||||
timeout=120,
|
||||
)
|
||||
assert command_result.returncode == 0, f"archivebox add failed:\nSTDOUT:\n{command_result.stdout}\nSTDERR:\n{command_result.stderr}"
|
||||
assert_browser_runtime_unchanged("archivebox add --plugins=archivewebpage")
|
||||
|
||||
capture = _get_archivewebpage_capture(initialized_archive, url)
|
||||
snapshot_host = f"{get_snapshot_subdomain(capture['snapshot_id'])}.archivebox.localhost:{port}"
|
||||
|
||||
@@ -83,6 +83,96 @@ def test_foreground_shutdown_can_request_cooperative_shutdown_without_raising():
|
||||
assert process.returncode == 130, (stdout, stderr)
|
||||
|
||||
|
||||
def test_cooperative_shutdown_during_child_poll_keeps_child_reapable():
|
||||
"""Interrupt real child polling at different points without stranding reaping.
|
||||
|
||||
Immediate KeyboardInterrupt stranded Popen's wait lock in real CI, leaving
|
||||
a zombie child that wait() could not reap. Exercise the cooperative policy
|
||||
used by the server with the same external signals and child cleanup.
|
||||
"""
|
||||
process = _start_signal_process(
|
||||
r"""
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import traceback
|
||||
from collections import Counter
|
||||
|
||||
from archivebox.core.shutdown_util import (
|
||||
foreground_shutdown_signals,
|
||||
raise_if_shutdown_requested,
|
||||
wait_popen_and_kill_children,
|
||||
)
|
||||
|
||||
print("READY", flush=True)
|
||||
interruption_sites = Counter()
|
||||
for attempt in range(512):
|
||||
child = subprocess.Popen(
|
||||
[sys.executable, "-c", "import time; time.sleep(60)"],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
interruption_site = None
|
||||
try:
|
||||
with foreground_shutdown_signals(first_signal_message=None, raise_on_first_signal=False) as state:
|
||||
try:
|
||||
os.write(sys.stdout.fileno(), b"POLLING\n")
|
||||
while True:
|
||||
raise_if_shutdown_requested()
|
||||
assert child.poll() is None
|
||||
except KeyboardInterrupt as interruption:
|
||||
assert state.signal_name == "SIGINT"
|
||||
frame = traceback.extract_tb(interruption.__traceback__)[-1]
|
||||
assert frame.name == "raise_if_shutdown_requested", frame
|
||||
interruption_site = f"{frame.name}:{frame.lineno}"
|
||||
interruption_sites[interruption_site] += 1
|
||||
# The child deliberately remains alive until this unchanged
|
||||
# kill-and-reap path asks it to stop.
|
||||
wait_popen_and_kill_children(child, [], timeout=0.001, kill_timeout=1.0)
|
||||
assert child.returncode == -signal.SIGKILL, (attempt, child.returncode)
|
||||
except BaseException as error:
|
||||
error.add_note(f"Interrupted child polling iteration: {attempt}, at {interruption_site}")
|
||||
raise
|
||||
finally:
|
||||
# Cleanup must also work if the assertion exposes a broken
|
||||
# Popen wait; preserve that failure while reaping the real PID.
|
||||
if child.returncode is None:
|
||||
try:
|
||||
os.kill(child.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
os.waitpid(child.pid, 0)
|
||||
except ChildProcessError:
|
||||
pass
|
||||
print(f"Interrupt locations: {dict(interruption_sites)}", file=sys.stderr, flush=True)
|
||||
print("REAPED:512", flush=True)
|
||||
""",
|
||||
)
|
||||
|
||||
try:
|
||||
# An external sender does not contend for the polling interpreter's
|
||||
# GIL. A Timer thread can bias delivery toward waitpid's GIL release,
|
||||
# missing other interruption points reached by real CLI Ctrl+C.
|
||||
assert process.stdout is not None
|
||||
sent = 0
|
||||
while sent < 512 and process.stdout.readline() == "POLLING\n":
|
||||
process.send_signal(signal.SIGINT)
|
||||
sent += 1
|
||||
stdout, stderr = process.communicate(timeout=30)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
|
||||
assert process.returncode == 0, (stdout, stderr)
|
||||
assert sent == 512, (sent, stdout, stderr)
|
||||
assert stdout == "REAPED:512\n", (stdout, stderr)
|
||||
print(stderr)
|
||||
|
||||
|
||||
def test_daemon_runner_signal_exit_is_unexpected_for_supervisor():
|
||||
process = _start_signal_process(
|
||||
"""
|
||||
|
||||
@@ -311,6 +311,38 @@ def test_responses_card_uses_relative_preview_urls_in_static_export(snapshot):
|
||||
assert "localhost" not in html
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.parametrize("status", ["queued", "started", "paused"])
|
||||
def test_empty_snapshot_keeps_live_progress_until_files_are_opened(snapshot, live_server, status):
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
port = urlsplit(live_server.url).port
|
||||
machine = Machine.current()
|
||||
machine.config = {**machine.config, "BASE_URL": f"http://archivebox.localhost:{port}"}
|
||||
machine.save(update_fields=["config"])
|
||||
snapshot.permissions = "public"
|
||||
snapshot.status = status
|
||||
snapshot.save(update_fields=["permissions", "status"])
|
||||
Path(snapshot.output_dir).mkdir(parents=True, exist_ok=True)
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
|
||||
page = browser.new_page()
|
||||
page.goto(f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}")
|
||||
assert page.locator("#progress-monitor").is_visible()
|
||||
assert page.locator("#progress-monitor .progress-content").is_visible()
|
||||
assert page.locator("#snapshot-empty-state").is_visible()
|
||||
assert page.locator("#main-frame").count() == 0
|
||||
|
||||
page.get_by_role("button", name=re.compile("^Other files,")).click()
|
||||
page.locator("#main-frame").wait_for()
|
||||
assert page.locator("#progress-monitor").count() == 0
|
||||
browser.close()
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_opening_files_from_queued_snapshot_stops_detached_progress_monitor(snapshot, live_server):
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Takeover utility tests and live command handoff flows."""
|
||||
|
||||
import os
|
||||
import json
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import pty
|
||||
import select
|
||||
import termios
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import psutil
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
assert_no_processes_for_data_dir,
|
||||
get_free_port,
|
||||
kill_processes_for_data_dir,
|
||||
cli_env,
|
||||
pid_is_alive,
|
||||
run_archivebox_cmd,
|
||||
start_archivebox_server,
|
||||
stop_archivebox_process,
|
||||
get_http_response,
|
||||
)
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
def _resolve_sonic_env(env: dict[str, str]) -> dict[str, str]:
|
||||
from abx_plugins import get_plugins_dir
|
||||
|
||||
config = Path(get_plugins_dir()) / "search_backend_sonic" / "config.json"
|
||||
result = subprocess.run(
|
||||
[
|
||||
str(Path(sys.executable).with_name("abxpkg")),
|
||||
"env",
|
||||
"--install",
|
||||
"--json",
|
||||
f"--lib={env['ABXPKG_LIB_DIR']}",
|
||||
f"--deps-from={config}:required_binaries",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
payload = {str(key): str(value) for key, value in json.loads(result.stdout).items()}
|
||||
assert Path(payload["SONIC_BINARY"]).is_file()
|
||||
return payload
|
||||
|
||||
|
||||
def _archive_pages_for_sqlite_reindexing(data_dir: Path, env: dict[str, str], root_url: str) -> None:
|
||||
add_env = dict(env)
|
||||
add_env["SEARCH_BACKEND_ENGINE"] = "ripgrep"
|
||||
_cmd_result = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=2",
|
||||
"--max-urls=20",
|
||||
"--crawl-max-size=50mb",
|
||||
"--plugins=wget,parse_html_urls",
|
||||
root_url,
|
||||
],
|
||||
cwd=data_dir,
|
||||
env=add_env,
|
||||
timeout=240,
|
||||
)
|
||||
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
|
||||
assert returncode == 0, stderr or stdout
|
||||
|
||||
with use_archivebox_db(data_dir):
|
||||
assert Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED).count() >= 1
|
||||
assert not ArchiveResult.objects.filter(plugin="search_backend_sqlite").exists()
|
||||
|
||||
|
||||
def _start_archivebox_shell(tmp_path: Path):
|
||||
process = run_archivebox_cmd(
|
||||
["manage", "shell"],
|
||||
cwd=tmp_path,
|
||||
env=cli_env(live=True),
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
capture_output=False,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
assert process.pid is not None
|
||||
assert process.stdin is not None
|
||||
assert pid_is_alive(process.pid)
|
||||
return process
|
||||
|
||||
|
||||
def _stop_archivebox_shells(processes) -> None:
|
||||
for process in processes:
|
||||
if process.stdin is not None and not process.stdin.closed:
|
||||
process.stdin.close()
|
||||
for process in processes:
|
||||
process.wait(timeout=20)
|
||||
assert not pid_is_alive(process.pid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interrupt_before_takeover", [False, True], ids=["running", "prompt"])
|
||||
@pytest.mark.parametrize("after_takeover", ["resume", "cancel-standby"])
|
||||
def test_interactive_add_takeover(initialized_archive, interrupt_before_takeover, after_takeover):
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 160))
|
||||
output = bytearray()
|
||||
add = None
|
||||
server = None
|
||||
port = get_free_port()
|
||||
env = cli_env(
|
||||
SEARCH_BACKEND_ENGINE="ripgrep",
|
||||
SEARCH_BACKEND_SONIC_ENABLED="False",
|
||||
CHROME_DELAY_AFTER_LOAD="60",
|
||||
CHROME_TIMEOUT="120",
|
||||
CHROME_HEADLESS="True",
|
||||
)
|
||||
installed = run_archivebox_cmd(["install", "chrome"], cwd=initialized_archive, env=env, timeout=600)
|
||||
assert installed.returncode == 0, installed.stderr or installed.stdout
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def active_hook_for(parent_pid):
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for hook in Process.objects.filter(
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
|
||||
status="running",
|
||||
):
|
||||
if hook.is_running and parent_pid in {parent.pid for parent in psutil.Process(hook.pid).parents()}:
|
||||
return hook
|
||||
return None
|
||||
|
||||
try:
|
||||
add = run_archivebox_cmd(
|
||||
["add", "--plugins=chrome", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
read_until(lambda: active_hook_for(add.pid) is not None)
|
||||
old_hook = active_hook_for(add.pid)
|
||||
assert old_hook is not None
|
||||
if interrupt_before_takeover:
|
||||
add.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
assert not pid_is_alive(old_hook.pid)
|
||||
|
||||
server = start_archivebox_server(initialized_archive, port=port, log_name="interrupt-takeover-server.log", env=env)
|
||||
read_until(lambda: b"A newer archivebox process took over" in output)
|
||||
assert add.poll() is None, output.decode(errors="replace")
|
||||
assert termios.tcgetattr(slave)[3] & termios.ICANON
|
||||
assert not pid_is_alive(old_hook.pid)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
|
||||
if after_takeover == "cancel-standby":
|
||||
add.send_signal(signal.SIGINT)
|
||||
read_until(lambda: add.poll() is not None)
|
||||
assert add.returncode == 130, output.decode(errors="replace")
|
||||
assert pid_is_alive(server.pid)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
else:
|
||||
read_until(lambda: active_hook_for(server.pid) is not None)
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
server = None
|
||||
read_until(lambda: active_hook_for(add.pid) is not None)
|
||||
resumed_hook = active_hook_for(add.pid)
|
||||
assert resumed_hook is not None and resumed_hook.pid != old_hook.pid
|
||||
prompt_offset = len(output)
|
||||
add.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output[prompt_offset:])
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
os.write(master, b"\r")
|
||||
read_until(lambda: add.poll() is not None)
|
||||
assert add.returncode == 0, output.decode(errors="replace")
|
||||
with use_archivebox_db(initialized_archive):
|
||||
assert Crawl.objects.get().status == Crawl.StatusChoices.SEALED
|
||||
assert b"Traceback" not in output
|
||||
finally:
|
||||
for process in (server, add):
|
||||
if process is not None and process.poll() is None:
|
||||
stop_archivebox_process(process, signal.SIGTERM)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
kill_processes_for_data_dir(initialized_archive)
|
||||
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
|
||||
@@ -0,0 +1,369 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Takeover utility tests and live command handoff flows."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import pty
|
||||
import select
|
||||
import termios
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
assert_no_processes_for_data_dir,
|
||||
get_free_port,
|
||||
kill_processes_for_data_dir,
|
||||
cli_env,
|
||||
pid_is_alive,
|
||||
run_archivebox_cmd,
|
||||
start_archivebox_server,
|
||||
stop_archivebox_process,
|
||||
get_http_response,
|
||||
wait_for_snapshot_capture,
|
||||
)
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
from .test_takeover_util_1 import (
|
||||
pytestmark as pytestmark,
|
||||
_resolve_sonic_env as _resolve_sonic_env,
|
||||
_archive_pages_for_sqlite_reindexing as _archive_pages_for_sqlite_reindexing,
|
||||
_start_archivebox_shell as _start_archivebox_shell,
|
||||
_stop_archivebox_shells as _stop_archivebox_shells,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("force_abort", [None, "typed", "signal"], ids=["graceful", "force-after-choice", "force-after-ctrl-c"])
|
||||
def test_interactive_add_borrowing_server_supervisor_can_abort(initialized_archive, force_abort):
|
||||
"""A server-owned supervisor must not make an interactive add worker noninteractive."""
|
||||
master, slave = pty.openpty()
|
||||
termios.tcsetwinsize(slave, (40, 160))
|
||||
output = bytearray()
|
||||
server = None
|
||||
add = None
|
||||
port = get_free_port()
|
||||
env = cli_env(
|
||||
SEARCH_BACKEND_ENGINE="ripgrep",
|
||||
SEARCH_BACKEND_SONIC_ENABLED="False",
|
||||
CHROME_DELAY_AFTER_LOAD="60",
|
||||
CHROME_TIMEOUT="120",
|
||||
CHROME_HEADLESS="True",
|
||||
)
|
||||
installed = run_archivebox_cmd(["install", "chrome"], cwd=initialized_archive, env=env, timeout=600)
|
||||
assert installed.returncode == 0, installed.stderr or installed.stdout
|
||||
|
||||
def read_until(predicate, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if select.select([master], [], [], 0.1)[0]:
|
||||
output.extend(os.read(master, 65536))
|
||||
if predicate():
|
||||
return
|
||||
raise AssertionError(output.decode(errors="replace"))
|
||||
|
||||
def active_hook():
|
||||
with use_archivebox_db(initialized_archive):
|
||||
for hook in Process.objects.filter(
|
||||
process_type=Process.TypeChoices.HOOK,
|
||||
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
|
||||
status="running",
|
||||
):
|
||||
if hook.is_running:
|
||||
return hook
|
||||
return None
|
||||
|
||||
try:
|
||||
server = start_archivebox_server(initialized_archive, port=port, log_name="add-borrows-server.log", env=env)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
add = run_archivebox_cmd(
|
||||
["add", "--plugins=chrome", "https://example.com"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
stdin=slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
wait=False,
|
||||
start_new_session=True,
|
||||
)
|
||||
read_until(lambda: active_hook() is not None)
|
||||
old_hook = active_hook()
|
||||
assert old_hook is not None
|
||||
add.send_signal(signal.SIGINT)
|
||||
read_until(lambda: b"Choice [skip]:" in output)
|
||||
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
assert add.poll() is None, output.decode(errors="replace")
|
||||
assert not pid_is_alive(old_hook.pid)
|
||||
|
||||
if force_abort:
|
||||
# Both abort choices must become forceable before the worker has
|
||||
# time to write its marker or finish the normal hook grace phase.
|
||||
if force_abort == "typed":
|
||||
os.write(master, b"a")
|
||||
else:
|
||||
add.send_signal(signal.SIGINT)
|
||||
read_until(lambda: termios.tcgetattr(slave)[3] & termios.ICANON)
|
||||
forced_at = time.monotonic()
|
||||
add.send_signal(signal.SIGINT)
|
||||
else:
|
||||
add.send_signal(signal.SIGINT)
|
||||
read_until(lambda: add.poll() is not None)
|
||||
if force_abort:
|
||||
assert time.monotonic() - forced_at < 2.0, output.decode(errors="replace")
|
||||
assert b"Forcing aborted crawl to exit now" in output
|
||||
assert add.returncode == 130, output.decode(errors="replace")
|
||||
assert pid_is_alive(server.pid)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
assert b"Traceback" not in output
|
||||
finally:
|
||||
for process in (add, server):
|
||||
if process is not None and process.poll() is None:
|
||||
stop_archivebox_process(process, signal.SIGTERM)
|
||||
os.close(slave)
|
||||
os.close(master)
|
||||
kill_processes_for_data_dir(initialized_archive)
|
||||
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
|
||||
|
||||
|
||||
def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive):
|
||||
sonic_port = get_free_port()
|
||||
lib_dir = initialized_archive / "lib"
|
||||
env = cli_env(
|
||||
live=True,
|
||||
ABXPKG_LIB_DIR=str(lib_dir),
|
||||
PLUGINS="search_backend_sonic",
|
||||
SEARCH_BACKEND_ENGINE="sonic",
|
||||
SEARCH_BACKEND_SONIC_PORT=str(sonic_port),
|
||||
)
|
||||
env.update(_resolve_sonic_env(env))
|
||||
env["PATH"] = os.pathsep.join((str(Path(sys.executable).parent), env["PATH"]))
|
||||
installed_binary = Path(env["SONIC_BINARY"]).resolve(strict=True)
|
||||
managed_sonic_dir = lib_dir / "bash" / "bin"
|
||||
managed_sonic_dir.mkdir(parents=True, exist_ok=True)
|
||||
managed_binary = managed_sonic_dir / "sonic"
|
||||
if installed_binary != managed_binary:
|
||||
shutil.copy2(installed_binary, managed_binary)
|
||||
config_result = run_archivebox_cmd(
|
||||
["config", "--set", f"SONIC_BINARY={managed_binary}"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert config_result.returncode == 0, config_result.stderr or config_result.stdout
|
||||
pinned_sonic_dir = initialized_archive / "pinned-sonic"
|
||||
pinned_sonic_dir.mkdir()
|
||||
sonic_binary = pinned_sonic_dir / "sonic"
|
||||
shutil.copy2(installed_binary, sonic_binary)
|
||||
env["SONIC_BINARY"] = str(sonic_binary)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["list", "--search=contents", "--csv=url", "not-indexed"],
|
||||
cwd=initialized_archive,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
config_match = re.search(r"Using supervisord config file: (\S+)", result.stderr)
|
||||
assert config_match, result.stderr
|
||||
worker_config = Path(config_match.group(1)).parent / "workers" / "worker_sonic.conf"
|
||||
worker_text = worker_config.read_text(encoding="utf-8")
|
||||
command = next(line.partition("=")[2] for line in worker_text.splitlines() if line.startswith("command="))
|
||||
assert shlex.split(command)[0] == str(sonic_binary), worker_text
|
||||
from archivebox.workers.supervisord_util import _sonic_worker_bind_target
|
||||
|
||||
assert _sonic_worker_bind_target({"name": "worker_sonic", "command": command}) == ("127.0.0.1", sonic_port)
|
||||
renamed_binary = initialized_archive / "renamed-sonic-binary"
|
||||
shutil.copy2(sonic_binary, renamed_binary)
|
||||
renamed_command = shlex.join([str(renamed_binary), *shlex.split(command)[1:]])
|
||||
assert _sonic_worker_bind_target({"name": "worker_sonic", "command": renamed_command}) == ("127.0.0.1", sonic_port)
|
||||
|
||||
relative_binary = pinned_sonic_dir / "sonic-alt"
|
||||
shutil.copy2(sonic_binary, relative_binary)
|
||||
relative_env = {
|
||||
**env,
|
||||
"SONIC_BINARY": relative_binary.name,
|
||||
"PATH": os.pathsep.join((str(pinned_sonic_dir), env["PATH"])),
|
||||
"SEARCH_BACKEND_SONIC_PORT": str(get_free_port()),
|
||||
}
|
||||
relative_result = run_archivebox_cmd(
|
||||
["list", "--search=contents", "--csv=url", "not-indexed"],
|
||||
cwd=initialized_archive,
|
||||
env=relative_env,
|
||||
timeout=60,
|
||||
)
|
||||
assert relative_result.returncode == 0, relative_result.stderr or relative_result.stdout
|
||||
relative_worker_text = worker_config.read_text(encoding="utf-8")
|
||||
relative_command = next(line.partition("=")[2] for line in relative_worker_text.splitlines() if line.startswith("command="))
|
||||
assert Path(shlex.split(relative_command)[0]).name == relative_binary.name, relative_worker_text
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
def test_behavior_foreground_add_keeps_existing_server_http_visible(tmp_path, initialized_archive, recursive_test_site):
|
||||
|
||||
port = get_free_port()
|
||||
env = cli_env(live=True, server=True, port=port, SEARCH_BACKEND_ENGINE="ripgrep")
|
||||
server = None
|
||||
try:
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-add.log", env=env)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
|
||||
add = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--depth=2",
|
||||
"--max-urls=10",
|
||||
"--plugins=wget,parse_html_urls",
|
||||
recursive_test_site["root_url"],
|
||||
],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
|
||||
assert add.returncode == 0, add.stderr or add.stdout
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=120)
|
||||
assert "Root" in captured_text
|
||||
finally:
|
||||
if server is not None:
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
def test_behavior_background_add_returns_and_server_archives_visible_url(tmp_path, initialized_archive, recursive_test_site):
|
||||
|
||||
port = get_free_port()
|
||||
env = cli_env(live=True, server=True, port=port, SEARCH_BACKEND_ENGINE="ripgrep")
|
||||
server = None
|
||||
try:
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-bg-add.log", env=env)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
|
||||
add = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--bg",
|
||||
"--depth=0",
|
||||
"--plugins=wget",
|
||||
recursive_test_site["root_url"],
|
||||
],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert add.returncode == 0, add.stderr or add.stdout
|
||||
assert "background runner will process" in add.stdout
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
|
||||
assert "Root" in captured_text
|
||||
finally:
|
||||
if server is not None:
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
def test_foreground_runner_owner_prefers_newer_update_over_server(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner
|
||||
|
||||
procs: list[subprocess.Popen[str]] = []
|
||||
try:
|
||||
for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE):
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
procs.append(proc)
|
||||
Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=process_type,
|
||||
worker_type=process_type,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=proc.pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
runtime_owner = runtime_stack_owner(data_dir=tmp_path)
|
||||
runner_owner = foreground_runner_owner(data_dir=tmp_path)
|
||||
|
||||
assert runtime_owner is not None
|
||||
assert runtime_owner.process_type == Process.TypeChoices.SERVER
|
||||
assert runner_owner is not None
|
||||
assert runner_owner.process_type == Process.TypeChoices.UPDATE
|
||||
assert runner_owner.pid == procs[-1].pid
|
||||
finally:
|
||||
_stop_archivebox_shells(procs)
|
||||
|
||||
|
||||
def test_runtime_stack_owner_ignores_supervised_orphan_runner(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner
|
||||
|
||||
procs: list[subprocess.Popen[str]] = []
|
||||
try:
|
||||
for _ in range(2):
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
procs.append(proc)
|
||||
|
||||
supervisor_row = Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.SUPERVISORD,
|
||||
worker_type="supervisord",
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=procs[0].pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
parent=supervisor_row,
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=procs[1].pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
assert runtime_stack_owner(data_dir=tmp_path) is None
|
||||
finally:
|
||||
_stop_archivebox_shells(procs)
|
||||
|
||||
|
||||
def test_foreign_machine_runner_only_warns(tmp_path, initialized_archive, capsys):
|
||||
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, live_runner_processes
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
foreign_machine = Machine.objects.create(
|
||||
guid="foreign-machine",
|
||||
hostname="foreign-host",
|
||||
hw_manufacturer="Test",
|
||||
hw_product="Test",
|
||||
hw_uuid="foreign-hardware",
|
||||
os_arch="x86_64",
|
||||
os_family="linux",
|
||||
os_platform="linux",
|
||||
os_release="test",
|
||||
os_kernel="test",
|
||||
)
|
||||
foreign_runner = Process.objects.create(
|
||||
machine=foreign_machine,
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
|
||||
pwd=str(tmp_path),
|
||||
pid=1,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
assert live_runner_processes(data_dir=tmp_path) == []
|
||||
foreign_runner.refresh_from_db()
|
||||
assert foreign_runner.status == Process.StatusChoices.RUNNING
|
||||
assert "Multiple orchestrators sharing a single collection is not officially supported" in capsys.readouterr().err
|
||||
@@ -0,0 +1,400 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Takeover utility tests and live command handoff flows."""
|
||||
|
||||
import signal
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
import psutil
|
||||
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
assert_no_processes_for_data_dir,
|
||||
get_free_port,
|
||||
kill_processes_for_data_dir,
|
||||
cli_env,
|
||||
pid_is_alive,
|
||||
run_archivebox_cmd,
|
||||
start_archivebox_server,
|
||||
stop_archivebox_process,
|
||||
supervisor_pid_from_log,
|
||||
get_http_response,
|
||||
wait_for_log_count,
|
||||
wait_for_pid_to_disappear,
|
||||
wait_for_snapshot_capture,
|
||||
worker_pid_from_log,
|
||||
)
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
from .test_takeover_util_1 import (
|
||||
pytestmark as pytestmark,
|
||||
_resolve_sonic_env as _resolve_sonic_env,
|
||||
_archive_pages_for_sqlite_reindexing as _archive_pages_for_sqlite_reindexing,
|
||||
_start_archivebox_shell as _start_archivebox_shell,
|
||||
_stop_archivebox_shells as _stop_archivebox_shells,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
def test_behavior_daemonized_server_restarts_cleanly_after_forced_stop(tmp_path, initialized_archive):
|
||||
|
||||
port = get_free_port()
|
||||
env = cli_env(live=True, server=True, port=port)
|
||||
try:
|
||||
first = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True)
|
||||
assert first.returncode == 0, first.stderr or first.stdout
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
second = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True)
|
||||
assert second.returncode == 0, second.stderr or second.stdout
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
finally:
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
def test_live_second_server_takes_over_existing_server_process(tmp_path, initialized_archive):
|
||||
|
||||
env = cli_env(live=True)
|
||||
port = get_free_port()
|
||||
first = None
|
||||
second = None
|
||||
try:
|
||||
first = start_archivebox_server(tmp_path, port=port, log_name="server-first.log", env=env)
|
||||
first_log = first.log_path
|
||||
second = start_archivebox_server(tmp_path, port=port, log_name="server-second.log", env=env)
|
||||
second_log = second.log_path
|
||||
|
||||
assert pid_is_alive(first.pid)
|
||||
first_text = first_log.read_text(encoding="utf-8", errors="replace")
|
||||
second_text = second_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert "A newer archivebox process took over the orchestrator, server" in first_text
|
||||
assert "Starting orchestrator, server" in second_text
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["status"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
first_resumes = first_log.read_text(encoding="utf-8", errors="replace").count("Other newer archivebox process")
|
||||
stop_archivebox_process(second, signal.SIGTERM)
|
||||
second = None
|
||||
wait_for_log_count(first_log, "Other newer archivebox process", first_resumes + 1, timeout=35)
|
||||
assert pid_is_alive(first.pid)
|
||||
finally:
|
||||
if second is not None:
|
||||
stop_archivebox_process(second, signal.SIGTERM)
|
||||
if first is not None:
|
||||
stop_archivebox_process(first, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(360)
|
||||
def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tmp_path, initialized_archive, recursive_test_site):
|
||||
env = cli_env(
|
||||
live=True,
|
||||
PLUGINS="wget,parse_html_urls,search_backend_sqlite,search_backend_sonic",
|
||||
SEARCH_BACKEND_ENGINE="sqlite",
|
||||
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
|
||||
)
|
||||
env.update(_resolve_sonic_env(env))
|
||||
_archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"])
|
||||
|
||||
port = get_free_port()
|
||||
server = None
|
||||
try:
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="server-real-sqlite-update.log", env=env)
|
||||
server_log = server.log_path
|
||||
supervisor_pid_before = supervisor_pid_from_log(server_log)
|
||||
daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne")
|
||||
runner_pid_before = worker_pid_from_log(server_log, "worker_runner")
|
||||
sonic_pid_before = worker_pid_from_log(server_log, "worker_sonic")
|
||||
supervisord_log = tmp_path / "logs" / "supervisord.log"
|
||||
runner_spawn_text = "spawned: 'worker_runner' with pid"
|
||||
runner_spawn_count = supervisord_log.read_text(encoding="utf-8", errors="replace").count(runner_spawn_text)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["update", "--index-only", "--batch-size=1"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert pid_is_alive(server.pid)
|
||||
assert supervisor_pid_from_log(server_log) == supervisor_pid_before
|
||||
assert pid_is_alive(daphne_pid_before)
|
||||
assert pid_is_alive(sonic_pid_before)
|
||||
assert worker_pid_from_log(server_log, "worker_daphne") == daphne_pid_before
|
||||
assert worker_pid_from_log(server_log, "worker_sonic") == sonic_pid_before
|
||||
assert "A newer archivebox process took over the orchestrator, server" not in server_log.read_text(
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
supervisord_text = supervisord_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert supervisord_text.count(runner_spawn_text) == runner_spawn_count
|
||||
assert worker_pid_from_log(server_log, "worker_runner") == runner_pid_before
|
||||
assert pid_is_alive(runner_pid_before)
|
||||
|
||||
with use_archivebox_db(tmp_path):
|
||||
indexed_results = list(
|
||||
ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True),
|
||||
)
|
||||
assert indexed_results
|
||||
assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results)
|
||||
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
server = None
|
||||
wait_for_pid_to_disappear(daphne_pid_before, timeout=20)
|
||||
wait_for_pid_to_disappear(sonic_pid_before, timeout=20)
|
||||
wait_for_pid_to_disappear(runner_pid_before, timeout=20)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
finally:
|
||||
if server is not None:
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(420)
|
||||
def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_archive):
|
||||
|
||||
env = cli_env(live=True)
|
||||
port = get_free_port()
|
||||
servers: list[subprocess.Popen[str]] = []
|
||||
server_pids: list[int] = []
|
||||
daphne_pids: list[int] = []
|
||||
runner_pids: list[int] = []
|
||||
try:
|
||||
for index in range(5):
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name=f"server-chaos-{index}.log", env=env)
|
||||
log_path = server.log_path
|
||||
servers.append(server)
|
||||
server_pids.append(server.pid)
|
||||
daphne_pids.append(worker_pid_from_log(log_path, "worker_daphne"))
|
||||
runner_pids.append(worker_pid_from_log(log_path, "worker_runner"))
|
||||
|
||||
if index > 0:
|
||||
previous_server = servers[index - 1]
|
||||
previous_log = (tmp_path / f"server-chaos-{index - 1}.log").read_text(encoding="utf-8", errors="replace")
|
||||
current_log = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
assert pid_is_alive(previous_server.pid)
|
||||
assert pid_is_alive(server_pids[index - 1])
|
||||
assert "A newer archivebox process took over the orchestrator, server" in previous_log
|
||||
assert "Starting orchestrator, server" in current_log
|
||||
wait_for_pid_to_disappear(daphne_pids[index - 1], timeout=15)
|
||||
wait_for_pid_to_disappear(runner_pids[index - 1], timeout=15)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["status"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
|
||||
assert pid_is_alive(servers[-1].pid)
|
||||
assert all(pid_is_alive(server.pid) for server in servers)
|
||||
# Inspect the actual server worker rather than requiring a host lsof
|
||||
# installation. Previous workers must be gone, and the newest worker
|
||||
# must own exactly one listening socket on our assigned port.
|
||||
assert all(not pid_is_alive(pid) for pid in daphne_pids[:-1])
|
||||
listeners = [
|
||||
connection
|
||||
for connection in psutil.Process(daphne_pids[-1]).net_connections(kind="tcp")
|
||||
if connection.status == psutil.CONN_LISTEN and connection.laddr.port == port
|
||||
]
|
||||
assert len(listeners) == 1, listeners
|
||||
|
||||
previous_log_path = tmp_path / "server-chaos-3.log"
|
||||
previous_takeovers = previous_log_path.read_text(encoding="utf-8", errors="replace").count(
|
||||
"Other newer archivebox process",
|
||||
)
|
||||
stop_archivebox_process(servers[-1], signal.SIGTERM)
|
||||
wait_for_log_count(previous_log_path, "Other newer archivebox process", previous_takeovers + 1, timeout=35)
|
||||
assert pid_is_alive(servers[3].pid)
|
||||
finally:
|
||||
for server in reversed(servers):
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(420)
|
||||
def test_live_background_add_survives_server_exit_and_foreground_run_reclaims(tmp_path, initialized_archive, recursive_test_site):
|
||||
env = cli_env(live=True, SEARCH_BACKEND_ENGINE="ripgrep")
|
||||
port = get_free_port()
|
||||
server = None
|
||||
server2 = None
|
||||
try:
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-1.log", env=env)
|
||||
server_log = server.log_path
|
||||
supervisor_pid_before = supervisor_pid_from_log(server_log)
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["update", "--index-only", "--batch-size=10"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=90,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert pid_is_alive(server.pid)
|
||||
assert pid_is_alive(supervisor_pid_before)
|
||||
assert supervisor_pid_from_log(server_log) == supervisor_pid_before
|
||||
|
||||
add = run_archivebox_cmd(
|
||||
[
|
||||
"add",
|
||||
"--bg",
|
||||
"--depth=1",
|
||||
"--max-urls=20",
|
||||
"--crawl-max-size=50mb",
|
||||
"--plugins=wget,parse_html_urls",
|
||||
recursive_test_site["root_url"],
|
||||
],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
assert add.returncode == 0, add.stderr or add.stdout
|
||||
assert "background runner will process" in add.stdout
|
||||
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
server = None
|
||||
assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
with use_archivebox_db(tmp_path):
|
||||
crawl_id = str(Crawl.objects.get().id)
|
||||
run = run_archivebox_cmd(
|
||||
["run", f"--crawl-id={crawl_id}"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
assert run.returncode == 0, run.stderr or run.stdout
|
||||
|
||||
server2 = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
|
||||
assert "Root" in captured_text
|
||||
stop_archivebox_process(server2, signal.SIGTERM)
|
||||
server2 = None
|
||||
with use_archivebox_db(tmp_path):
|
||||
crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at"))
|
||||
snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at"))
|
||||
bad_results = list(
|
||||
ArchiveResult.objects.filter(
|
||||
status__in=[
|
||||
ArchiveResult.StatusChoices.FAILED,
|
||||
ArchiveResult.StatusChoices.SKIPPED,
|
||||
],
|
||||
).values_list("plugin", "status", "output_str"),
|
||||
)
|
||||
assert crawls
|
||||
assert snapshots
|
||||
assert all(status == Crawl.StatusChoices.SEALED for status, _retry_at in crawls)
|
||||
assert all(status == Snapshot.StatusChoices.SEALED for _url, status, _retry_at in snapshots)
|
||||
assert not bad_results
|
||||
finally:
|
||||
for proc in (server, server2):
|
||||
if proc is not None:
|
||||
stop_archivebox_process(proc, signal.SIGTERM, timeout=10)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
# Utility-level takeover selection tests.
|
||||
|
||||
|
||||
def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import runtime_stack_owner
|
||||
|
||||
procs: list[subprocess.Popen[str]] = []
|
||||
try:
|
||||
for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER):
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
procs.append(proc)
|
||||
Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=process_type,
|
||||
worker_type=process_type,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=proc.pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
owner = runtime_stack_owner(data_dir=tmp_path)
|
||||
|
||||
assert owner is not None
|
||||
assert owner.process_type == Process.TypeChoices.SERVER
|
||||
assert owner.pid == procs[-1].pid
|
||||
finally:
|
||||
_stop_archivebox_shells(procs)
|
||||
|
||||
|
||||
def test_foreground_runner_owner_prefers_newer_server_over_update(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner
|
||||
|
||||
procs: list[subprocess.Popen[str]] = []
|
||||
try:
|
||||
for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER):
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
procs.append(proc)
|
||||
Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=process_type,
|
||||
worker_type=process_type,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=proc.pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
runtime_owner = runtime_stack_owner(data_dir=tmp_path)
|
||||
runner_owner = foreground_runner_owner(data_dir=tmp_path)
|
||||
|
||||
assert runtime_owner is not None
|
||||
assert runtime_owner.process_type == Process.TypeChoices.SERVER
|
||||
assert runner_owner is not None
|
||||
assert runner_owner.process_type == Process.TypeChoices.SERVER
|
||||
assert runner_owner.pid == procs[-1].pid
|
||||
finally:
|
||||
_stop_archivebox_shells(procs)
|
||||
|
||||
|
||||
def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exists(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner
|
||||
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
try:
|
||||
runner_row = Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=proc.pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
owner = runtime_stack_owner(data_dir=tmp_path)
|
||||
|
||||
assert owner is not None
|
||||
assert owner.id == runner_row.id
|
||||
finally:
|
||||
_stop_archivebox_shells([proc])
|
||||
@@ -0,0 +1,377 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Takeover utility tests and live command handoff flows."""
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
import psutil
|
||||
|
||||
from archivebox.core.models import ArchiveResult
|
||||
from archivebox.machine.models import Process
|
||||
from archivebox.tests.conftest import (
|
||||
assert_no_processes_for_data_dir,
|
||||
get_free_port,
|
||||
kill_processes_for_data_dir,
|
||||
cli_env,
|
||||
pid_is_alive,
|
||||
run_archivebox_cmd,
|
||||
start_archivebox_server,
|
||||
stop_archivebox_process,
|
||||
supervisor_pid_from_log,
|
||||
get_http_response,
|
||||
wait_for_log,
|
||||
wait_for_log_pattern,
|
||||
wait_for_pid_to_disappear,
|
||||
wait_for_worker_pid_from_log,
|
||||
worker_pid_from_log,
|
||||
)
|
||||
from archivebox.tests.test_orm_helpers import use_archivebox_db
|
||||
|
||||
from .test_takeover_util_1 import (
|
||||
pytestmark as pytestmark,
|
||||
_resolve_sonic_env as _resolve_sonic_env,
|
||||
_archive_pages_for_sqlite_reindexing as _archive_pages_for_sqlite_reindexing,
|
||||
_start_archivebox_shell as _start_archivebox_shell,
|
||||
_stop_archivebox_shells as _stop_archivebox_shells,
|
||||
)
|
||||
|
||||
|
||||
def test_pid_is_alive_treats_unreaped_archivebox_cli_as_exited(tmp_path, initialized_archive):
|
||||
proc = run_archivebox_cmd(
|
||||
["version"],
|
||||
cwd=tmp_path,
|
||||
default_cli_env=True,
|
||||
disable_extractors=True,
|
||||
wait=False,
|
||||
)
|
||||
try:
|
||||
os.waitid(os.P_PID, proc.pid, os.WEXITED | os.WNOWAIT)
|
||||
assert psutil.Process(proc.pid).status() == psutil.STATUS_ZOMBIE
|
||||
assert not pid_is_alive(proc.pid)
|
||||
finally:
|
||||
proc.wait(timeout=5)
|
||||
|
||||
|
||||
@pytest.mark.timeout(360)
|
||||
def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_path, initialized_archive, recursive_test_site):
|
||||
env = cli_env(
|
||||
live=True,
|
||||
PLUGINS="wget,parse_html_urls,search_backend_sqlite",
|
||||
SEARCH_BACKEND_ENGINE="sqlite",
|
||||
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
|
||||
)
|
||||
env.update(_resolve_sonic_env(env))
|
||||
root_url = recursive_test_site["root_url"]
|
||||
_archive_pages_for_sqlite_reindexing(tmp_path, env, root_url)
|
||||
|
||||
port = get_free_port()
|
||||
server = None
|
||||
try:
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-update.log", env=env)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
|
||||
update = run_archivebox_cmd(
|
||||
["update", "--index-only", "--batch-size=1"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
|
||||
assert update.returncode == 0, update.stderr or update.stdout
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
|
||||
search = run_archivebox_cmd(
|
||||
["list", "--search=contents", "--csv=url", "Root"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert search.returncode == 0, search.stderr or search.stdout
|
||||
assert root_url in search.stdout
|
||||
finally:
|
||||
if server is not None:
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(420)
|
||||
def test_behavior_update_yields_to_server_then_finishes_visible_indexing(tmp_path, initialized_archive, recursive_test_site):
|
||||
env = cli_env(
|
||||
live=True,
|
||||
PLUGINS="wget,parse_html_urls,search_backend_sqlite",
|
||||
SEARCH_BACKEND_ENGINE="sqlite",
|
||||
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
|
||||
)
|
||||
env.update(_resolve_sonic_env(env))
|
||||
root_url = recursive_test_site["root_url"]
|
||||
_archive_pages_for_sqlite_reindexing(tmp_path, env, root_url)
|
||||
|
||||
port = get_free_port()
|
||||
update_proc = None
|
||||
server = None
|
||||
try:
|
||||
update_log = tmp_path / "behavior-update-yields.log"
|
||||
update_log_handle = update_log.open("w", encoding="utf-8")
|
||||
update_proc = run_archivebox_cmd(
|
||||
["update", "--index-only", "--batch-size=1"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
stdout=update_log_handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
update_log_handle.close()
|
||||
wait_for_log(update_log, "[*] Reindexing", timeout=90)
|
||||
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-takes-update.log", env=env)
|
||||
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
|
||||
wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90)
|
||||
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
server = None
|
||||
update_proc.wait(timeout=180)
|
||||
update_text = update_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert update_proc.returncode == 0, update_text
|
||||
|
||||
search = run_archivebox_cmd(
|
||||
["list", "--search=contents", "--csv=url", "Root"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
assert search.returncode == 0, search.stderr or search.stdout
|
||||
assert root_url in search.stdout
|
||||
finally:
|
||||
if update_proc is not None:
|
||||
stop_archivebox_process(update_proc, signal.SIGTERM)
|
||||
if server is not None:
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(180)
|
||||
def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, initialized_archive):
|
||||
|
||||
env = cli_env(live=True)
|
||||
port = get_free_port()
|
||||
server = None
|
||||
try:
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="server-update-owner.log", env=env)
|
||||
server_log = server.log_path
|
||||
supervisor_pid_before = supervisor_pid_from_log(server_log)
|
||||
daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne")
|
||||
|
||||
result = run_archivebox_cmd(
|
||||
["update", "--index-only", "--before=0"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
timeout=90,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr or result.stdout
|
||||
assert pid_is_alive(server.pid)
|
||||
assert pid_is_alive(supervisor_pid_before)
|
||||
assert pid_is_alive(daphne_pid_before)
|
||||
assert supervisor_pid_from_log(server_log) == supervisor_pid_before
|
||||
assert worker_pid_from_log(server_log, "worker_daphne") == daphne_pid_before
|
||||
assert "A newer archivebox process took over the orchestrator, server" not in server_log.read_text(
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
finally:
|
||||
if server is not None:
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
@pytest.mark.timeout(420)
|
||||
def test_live_update_yields_to_server_then_reclaims_real_sqlite_indexing(tmp_path, initialized_archive, recursive_test_site):
|
||||
env = cli_env(
|
||||
live=True,
|
||||
PLUGINS="wget,parse_html_urls,search_backend_sqlite,search_backend_sonic",
|
||||
SEARCH_BACKEND_ENGINE="sqlite",
|
||||
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
|
||||
)
|
||||
env.update(_resolve_sonic_env(env))
|
||||
_archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"])
|
||||
|
||||
port = get_free_port()
|
||||
update_proc = None
|
||||
server = None
|
||||
try:
|
||||
update_log = tmp_path / "update-real-sqlite-owner.log"
|
||||
update_log_handle = update_log.open("w", encoding="utf-8")
|
||||
update_proc = run_archivebox_cmd(
|
||||
["update", "--index-only", "--batch-size=1"],
|
||||
cwd=tmp_path,
|
||||
env=env,
|
||||
stdout=update_log_handle,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
wait=False,
|
||||
)
|
||||
update_log_handle.close()
|
||||
wait_for_log(update_log, "[*] Reindexing", timeout=90)
|
||||
update_supervisor_match = wait_for_log_pattern(update_log, r"Supervisord connected \(pid=(\d+)\)", timeout=90)
|
||||
update_supervisor_pid_before = int(update_supervisor_match.group(1))
|
||||
update_sonic_pid_before = wait_for_worker_pid_from_log(update_log, "worker_sonic", timeout=90)
|
||||
assert pid_is_alive(update_supervisor_pid_before)
|
||||
assert pid_is_alive(update_sonic_pid_before)
|
||||
assert "worker_runner_update_" not in update_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert "worker_daphne" not in update_log.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
server = start_archivebox_server(tmp_path, port=port, log_name="server-takes-real-sqlite-update.log", env=env)
|
||||
server_log = server.log_path
|
||||
wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90)
|
||||
assert pid_is_alive(update_proc.pid)
|
||||
assert pid_is_alive(server.pid)
|
||||
server_text = server_log.read_text(encoding="utf-8", errors="replace")
|
||||
# The older update process can yield orchestrator ownership just before
|
||||
# the server logs its takeover, but sonic must always move to the server.
|
||||
assert (
|
||||
"Taking over orchestrator, sonic from older existing archivebox process" in server_text
|
||||
or "Taking over sonic from older existing archivebox process" in server_text
|
||||
)
|
||||
assert "worker_daphne" in server_text
|
||||
assert "worker_sonic" in server_text
|
||||
server_daphne_pid = worker_pid_from_log(server_log, "worker_daphne")
|
||||
server_runner_pid = worker_pid_from_log(server_log, "worker_runner")
|
||||
server_sonic_pid = worker_pid_from_log(server_log, "worker_sonic")
|
||||
assert pid_is_alive(server_daphne_pid)
|
||||
assert pid_is_alive(server_runner_pid)
|
||||
assert pid_is_alive(server_sonic_pid)
|
||||
wait_for_pid_to_disappear(update_supervisor_pid_before, timeout=30)
|
||||
wait_for_pid_to_disappear(update_sonic_pid_before, timeout=30)
|
||||
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
server = None
|
||||
update_proc.wait(timeout=180)
|
||||
update_text = update_log.read_text(encoding="utf-8", errors="replace")
|
||||
assert update_proc.returncode == 0, update_text
|
||||
wait_for_pid_to_disappear(server_daphne_pid, timeout=20)
|
||||
wait_for_pid_to_disappear(server_runner_pid, timeout=20)
|
||||
wait_for_pid_to_disappear(server_sonic_pid, timeout=20)
|
||||
|
||||
with use_archivebox_db(tmp_path):
|
||||
indexed_results = list(
|
||||
ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True),
|
||||
)
|
||||
assert indexed_results
|
||||
assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
finally:
|
||||
if update_proc is not None:
|
||||
stop_archivebox_process(update_proc, signal.SIGTERM)
|
||||
if server is not None:
|
||||
stop_archivebox_process(server, signal.SIGTERM)
|
||||
kill_processes_for_data_dir(tmp_path)
|
||||
assert_no_processes_for_data_dir(tmp_path, timeout=12)
|
||||
|
||||
|
||||
def test_runtime_stack_owner_keeps_server_over_newer_update(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import runtime_stack_owner
|
||||
|
||||
procs: list[subprocess.Popen[str]] = []
|
||||
try:
|
||||
for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE):
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
procs.append(proc)
|
||||
Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=process_type,
|
||||
worker_type=process_type,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=proc.pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
owner = runtime_stack_owner(data_dir=tmp_path)
|
||||
|
||||
assert owner is not None
|
||||
assert owner.process_type == Process.TypeChoices.SERVER
|
||||
assert owner.pid == procs[0].pid
|
||||
finally:
|
||||
_stop_archivebox_shells(procs)
|
||||
|
||||
|
||||
def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner
|
||||
|
||||
procs: list[subprocess.Popen[str]] = []
|
||||
try:
|
||||
for process_type, worker_type in (
|
||||
(Process.TypeChoices.SERVER, ""),
|
||||
(Process.TypeChoices.ORCHESTRATOR, RUNNER_ACTIVE_WORKER_TYPE),
|
||||
):
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
procs.append(proc)
|
||||
Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=process_type,
|
||||
worker_type=worker_type,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=proc.pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
|
||||
owner = runtime_stack_owner(data_dir=tmp_path)
|
||||
|
||||
assert owner is not None
|
||||
assert owner.process_type == Process.TypeChoices.SERVER
|
||||
assert owner.pid == procs[0].pid
|
||||
finally:
|
||||
_stop_archivebox_shells(procs)
|
||||
|
||||
|
||||
def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path, initialized_archive):
|
||||
from archivebox.machine.models import Machine
|
||||
from archivebox.core.takeover_util import runtime_stack_owner
|
||||
|
||||
procs: list[subprocess.Popen[str]] = []
|
||||
older_row = None
|
||||
newer_row = None
|
||||
try:
|
||||
for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER):
|
||||
proc = _start_archivebox_shell(tmp_path)
|
||||
procs.append(proc)
|
||||
row = Process.objects.create(
|
||||
machine=Machine.current(),
|
||||
process_type=process_type,
|
||||
worker_type=process_type,
|
||||
pwd=str(tmp_path),
|
||||
cmd=[],
|
||||
pid=proc.pid,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
)
|
||||
if process_type == Process.TypeChoices.UPDATE:
|
||||
older_row = row
|
||||
else:
|
||||
newer_row = row
|
||||
|
||||
assert older_row is not None
|
||||
assert newer_row is not None
|
||||
|
||||
os.kill(procs[-1].pid, signal.SIGTERM)
|
||||
procs[-1].wait(timeout=20)
|
||||
|
||||
owner = runtime_stack_owner(data_dir=tmp_path)
|
||||
|
||||
assert owner is None
|
||||
newer_row.refresh_from_db()
|
||||
assert newer_row.status == Process.StatusChoices.EXITED
|
||||
older_row.refresh_from_db()
|
||||
assert older_row.status == Process.StatusChoices.RUNNING
|
||||
finally:
|
||||
_stop_archivebox_shells(procs)
|
||||
@@ -209,10 +209,13 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s
|
||||
_stop_worker(tmp_path, "worker_runner")
|
||||
assert _worker_state(tmp_path, "worker_runner") != "RUNNING"
|
||||
|
||||
session, csrf_token = _login_to_add_view(port)
|
||||
session, _add_page, csrf_token = _login_to_add_view(port)
|
||||
response = session.post(
|
||||
f"http://admin.archivebox.localhost:{port}/add/",
|
||||
headers={"Referer": f"http://admin.archivebox.localhost:{port}/add/"},
|
||||
f"http://127.0.0.1:{port}/add/",
|
||||
headers={
|
||||
"Host": f"admin.archivebox.localhost:{port}",
|
||||
"Referer": f"http://admin.archivebox.localhost:{port}/add/",
|
||||
},
|
||||
data={
|
||||
"url": recursive_test_site["root_url"],
|
||||
"depth": "0",
|
||||
@@ -246,19 +249,26 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s
|
||||
stop_server(tmp_path)
|
||||
|
||||
|
||||
def _login_to_add_view(port: int) -> tuple[requests.Session, str]:
|
||||
def _login_to_add_view(port: int) -> tuple[requests.Session, requests.Response, str]:
|
||||
# Browsers resolve *.localhost themselves; system DNS in stock containers
|
||||
# need not. Loopback transport with the real Host/Referer preserves virtual
|
||||
# host routing, session cookies and CSRF checks without depending on DNS.
|
||||
session = requests.Session()
|
||||
get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
|
||||
login_page = session.get(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
f"http://127.0.0.1:{port}/admin/login/",
|
||||
headers={"Host": f"admin.archivebox.localhost:{port}"},
|
||||
timeout=10,
|
||||
)
|
||||
assert login_page.status_code == 200
|
||||
csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
|
||||
assert csrf_match, login_page.text[:500]
|
||||
login_response = session.post(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
headers={"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/"},
|
||||
f"http://127.0.0.1:{port}/admin/login/",
|
||||
headers={
|
||||
"Host": f"admin.archivebox.localhost:{port}",
|
||||
"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
},
|
||||
data={
|
||||
"username": "apitestadmin",
|
||||
"password": "testpass123",
|
||||
@@ -270,12 +280,13 @@ def _login_to_add_view(port: int) -> tuple[requests.Session, str]:
|
||||
)
|
||||
assert login_response.status_code in (302, 303), login_response.text
|
||||
add_page, csrf_token = _get_add_view(session, port, host=f"admin.archivebox.localhost:{port}")
|
||||
return session, csrf_token
|
||||
return session, add_page, csrf_token
|
||||
|
||||
|
||||
def _get_add_view(session: requests.Session, port: int, *, host: str) -> tuple[requests.Response, str]:
|
||||
add_page = session.get(
|
||||
f"http://{host}/add/",
|
||||
f"http://127.0.0.1:{port}/add/",
|
||||
headers={"Host": host},
|
||||
timeout=10,
|
||||
)
|
||||
assert add_page.status_code == 200
|
||||
@@ -612,33 +623,14 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit
|
||||
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
session = requests.Session()
|
||||
get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
|
||||
login_page = session.get(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
timeout=10,
|
||||
)
|
||||
assert login_page.status_code == 200
|
||||
csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
|
||||
assert csrf_match, login_page.text[:500]
|
||||
login_response = session.post(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
headers={"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/"},
|
||||
data={
|
||||
"username": "apitestadmin",
|
||||
"password": "testpass123",
|
||||
"csrfmiddlewaretoken": csrf_match.group(1),
|
||||
"next": "/add/",
|
||||
},
|
||||
timeout=10,
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert login_response.status_code in (302, 303), login_response.text
|
||||
_add_page, add_csrf_token = _get_add_view(session, port, host=f"admin.archivebox.localhost:{port}")
|
||||
session, _add_page, add_csrf_token = _login_to_add_view(port)
|
||||
|
||||
response = session.post(
|
||||
f"http://admin.archivebox.localhost:{port}/add/",
|
||||
headers={"Referer": f"http://admin.archivebox.localhost:{port}/add/"},
|
||||
f"http://127.0.0.1:{port}/add/",
|
||||
headers={
|
||||
"Host": f"admin.archivebox.localhost:{port}",
|
||||
"Referer": f"http://admin.archivebox.localhost:{port}/add/",
|
||||
},
|
||||
data={
|
||||
"url": recursive_test_site["root_url"],
|
||||
"depth": "0",
|
||||
@@ -686,37 +678,18 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv
|
||||
|
||||
try:
|
||||
start_archivebox_server(tmp_path, env=env, port=port)
|
||||
session = requests.Session()
|
||||
get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
|
||||
login_page = session.get(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
timeout=10,
|
||||
)
|
||||
assert login_page.status_code == 200
|
||||
csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
|
||||
assert csrf_match, login_page.text[:500]
|
||||
login_response = session.post(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/login/",
|
||||
headers={"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/"},
|
||||
data={
|
||||
"username": "apitestadmin",
|
||||
"password": "testpass123",
|
||||
"csrfmiddlewaretoken": csrf_match.group(1),
|
||||
"next": "/add/",
|
||||
},
|
||||
timeout=10,
|
||||
allow_redirects=False,
|
||||
)
|
||||
assert login_response.status_code in (302, 303), login_response.text
|
||||
add_page, add_csrf_token = _get_add_view(session, port, host=f"admin.archivebox.localhost:{port}")
|
||||
session, add_page, add_csrf_token = _login_to_add_view(port)
|
||||
assert add_page.status_code == 200
|
||||
assert 'name="depth"' in add_page.text
|
||||
assert 'name="url"' in add_page.text
|
||||
_stop_worker(tmp_path, "worker_runner")
|
||||
|
||||
response = session.post(
|
||||
f"http://admin.archivebox.localhost:{port}/add/",
|
||||
headers={"Referer": f"http://admin.archivebox.localhost:{port}/add/"},
|
||||
f"http://127.0.0.1:{port}/add/",
|
||||
headers={
|
||||
"Host": f"admin.archivebox.localhost:{port}",
|
||||
"Referer": f"http://admin.archivebox.localhost:{port}/add/",
|
||||
},
|
||||
data={
|
||||
"url": recursive_test_site["root_url"],
|
||||
"depth": "2",
|
||||
@@ -781,14 +754,16 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv
|
||||
assert "active_crawls" in progress.json()
|
||||
|
||||
index_page = requests.get(
|
||||
f"http://web.archivebox.localhost:{port}/public/",
|
||||
f"http://127.0.0.1:{port}/public/",
|
||||
headers={"Host": f"web.archivebox.localhost:{port}"},
|
||||
timeout=10,
|
||||
)
|
||||
assert index_page.status_code == 200
|
||||
assert recursive_test_site["root_url"] in index_page.text
|
||||
|
||||
snapshot_admin = session.get(
|
||||
f"http://admin.archivebox.localhost:{port}/admin/core/snapshot/",
|
||||
f"http://127.0.0.1:{port}/admin/core/snapshot/",
|
||||
headers={"Host": f"admin.archivebox.localhost:{port}"},
|
||||
timeout=10,
|
||||
)
|
||||
assert snapshot_admin.status_code == 200
|
||||
|
||||
@@ -154,7 +154,7 @@ def running_process_record(initialized_archive):
|
||||
try:
|
||||
os_process = psutil.Process(popen.pid)
|
||||
process = Process.objects.create(
|
||||
machine=Machine.current(refresh=True),
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
pwd=str(initialized_archive),
|
||||
cmd=cmd,
|
||||
|
||||
@@ -35,7 +35,7 @@ def admin_request(admin_user):
|
||||
|
||||
@pytest.fixture
|
||||
def machine():
|
||||
return Machine.current(refresh=True)
|
||||
return Machine.current()
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
"""Live progress UI tests."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone as dt_timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
@@ -15,12 +19,48 @@ from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
|
||||
from archivebox.tests.conftest import ADMIN_TEST_HOST
|
||||
from archivebox.tests.conftest import cli_env, resolve_abxpkg_binary_env, run_archivebox_cmd
|
||||
from archivebox.tests.conftest import REPO_ROOT, cli_env, resolve_abxpkg_binary_env, resolve_abxpkg_chrome_env, run_archivebox_cmd
|
||||
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_screenshot_command(snapshot, live_server, cached_abxpkg_lib_dir, tmp_path):
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
port = urlsplit(live_server.url).port
|
||||
machine = Machine.current()
|
||||
machine.config = {**machine.config, "BASE_URL": f"http://archivebox.localhost:{port}"}
|
||||
machine.save(update_fields=["config"])
|
||||
snapshot.url = "https://sweeting.me"
|
||||
snapshot.permissions = "public"
|
||||
snapshot.save(update_fields=["url", "permissions"])
|
||||
Path(snapshot.output_dir).mkdir(parents=True, exist_ok=True)
|
||||
binary_env = resolve_abxpkg_chrome_env(cached_abxpkg_lib_dir)
|
||||
variants = [
|
||||
{"path": str(tmp_path / f"{name}.png"), "width": width, "height": height}
|
||||
for name, width, height in [("desktop", 1600, 1000), ("tablet", 1024, 1366), ("mobile", 390, 844)]
|
||||
]
|
||||
env = {
|
||||
**os.environ,
|
||||
**binary_env,
|
||||
"NODE_PATH": str(cached_abxpkg_lib_dir / "pnpm/packages/chrome/node_modules"),
|
||||
"SCREENSHOT_HOST_RESOLVER_RULES": "MAP *.archivebox.localhost 127.0.0.1",
|
||||
"SCREENSHOT_REQUIRE_LIVE_PROGRESS": "1",
|
||||
"SCREENSHOT_VARIANTS_JSON": json.dumps(variants),
|
||||
}
|
||||
command = [
|
||||
binary_env["NODE_BINARY"],
|
||||
str(REPO_ROOT / "bin/take_screenshot.js"),
|
||||
f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}",
|
||||
variants[0]["path"],
|
||||
]
|
||||
return command, env, variants
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def painted_http_server():
|
||||
"""Serve a complete static page so CDP stops emitting changed frames."""
|
||||
@@ -141,6 +181,249 @@ def real_crawl_setup_process(snapshot, hermetic_lib_dir):
|
||||
|
||||
|
||||
class TestLiveProgressView:
|
||||
def test_live_progress_uses_existing_indexes_and_cached_totals(self, client, admin_user, crawl, snapshot):
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
Crawl.objects.filter(pk=crawl.pk).update(status=Crawl.StatusChoices.STARTED)
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.SEALED, downloaded_at=None, output_size=1234)
|
||||
ArchiveResult.objects.create(snapshot=snapshot, plugin="search_backend_sonic", status=ArchiveResult.StatusChoices.QUEUED)
|
||||
client.force_login(admin_user)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
|
||||
query_sql = [query["sql"] for query in queries]
|
||||
assert response.status_code == 200, response.content
|
||||
payload = response.json()
|
||||
active_crawl = next(item for item in payload["active_crawls"] if item["id"] == str(crawl.pk))
|
||||
assert active_crawl["total_snapshots"] == 1
|
||||
assert active_crawl["crawl_output_size"] is None
|
||||
assert active_crawl["crawl_output_size_display"] == "…"
|
||||
deadline = time.monotonic() + 5
|
||||
while active_crawl["crawl_output_size"] is None and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200, response.content
|
||||
active_crawl = next(item for item in response.json()["active_crawls"] if item["id"] == str(crawl.pk))
|
||||
assert active_crawl["cancelled_snapshots"] == 1
|
||||
assert active_crawl["crawl_output_size"] == 1234
|
||||
assert payload["indexing_queued"] == 1
|
||||
assert not any("SUM(" in sql or 'COUNT("core_snapshot"."downloaded_at")' in sql for sql in query_sql)
|
||||
if connection.vendor == "sqlite":
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT name FROM sqlite_master WHERE type='index'")
|
||||
indexes = {row[0] for row in cursor.fetchall()}
|
||||
assert "archiveresult_progress_idx" not in indexes
|
||||
assert "snapshot_progress_totals_idx" not in indexes
|
||||
aggregates = [sql for sql in query_sql if "GROUP BY" in sql]
|
||||
assert aggregates
|
||||
with connection.cursor() as cursor:
|
||||
for sql in aggregates:
|
||||
cursor.execute("EXPLAIN QUERY PLAN " + sql)
|
||||
plan = " ".join(str(row) for row in cursor.fetchall())
|
||||
if 'FROM "core_snapshot"' in sql:
|
||||
assert "COVERING INDEX" in plan, plan
|
||||
else:
|
||||
# Both existing status indexes can satisfy this lookup.
|
||||
# Require a status-keyed search, not one planner choice's
|
||||
# index name; a full scan must still fail below.
|
||||
assert re.search(r"SEARCH core_archiveresult USING (?:COVERING )?INDEX \S+ \(status=\?", plan), plan
|
||||
assert "SCAN " not in plan, plan
|
||||
|
||||
# Size/cancellation samples are shared by tabs while status counts stay
|
||||
# current on every request.
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(output_size=4321)
|
||||
with CaptureQueriesContext(connection) as cached_queries:
|
||||
for _ in range(5):
|
||||
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200, response.content
|
||||
cached_crawl = next(item for item in response.json()["active_crawls"] if item["id"] == str(crawl.pk))
|
||||
assert cached_crawl["crawl_output_size"] == 1234
|
||||
assert not any("SUM(" in query["sql"] for query in cached_queries)
|
||||
|
||||
def test_live_progress_samples_batches_without_leaking_other_snapshots(self, client, admin_user, crawl, snapshot):
|
||||
from datetime import timedelta
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
Crawl.objects.filter(pk=crawl.pk).update(status=Crawl.StatusChoices.STARTED)
|
||||
now = timezone.now()
|
||||
Snapshot.objects.bulk_create(
|
||||
[
|
||||
Snapshot(
|
||||
crawl=crawl,
|
||||
url=f"https://example.com/batch/{i}",
|
||||
timestamp=str(now.timestamp() + i + 1),
|
||||
status=Snapshot.StatusChoices.SEALED,
|
||||
downloaded_at=now if i % 2 else None,
|
||||
output_size=i + 1,
|
||||
)
|
||||
for i in range(305)
|
||||
],
|
||||
)
|
||||
# Exercise both a timestamp tie crossing batch boundaries and advancing
|
||||
# to a new timestamp. A started snapshot must not enter sealed totals.
|
||||
Snapshot.objects.filter(crawl=crawl, status=Snapshot.StatusChoices.SEALED).update(modified_at=now)
|
||||
Snapshot.objects.filter(crawl=crawl, output_size__gt=250).update(modified_at=now + timedelta(seconds=1))
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(output_size=9999)
|
||||
client.force_login(admin_user)
|
||||
deadline = time.monotonic() + 5
|
||||
while True:
|
||||
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200, response.content
|
||||
totals = next(item for item in response.json()["active_crawls"] if item["id"] == str(crawl.pk))
|
||||
if totals["crawl_output_size"] is not None or time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert totals["crawl_output_size"] == sum(range(1, 306))
|
||||
assert totals["cancelled_snapshots"] == 153
|
||||
assert totals["total_snapshots"] == 306
|
||||
assert totals["completed_snapshots"] == 305
|
||||
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.SEALED, config={"PERMISSIONS": "public"})
|
||||
client.logout()
|
||||
deadline = time.monotonic() + 5
|
||||
while True:
|
||||
response = client.get(reverse("live_progress"), {"snapshot_id": str(snapshot.pk)}, HTTP_HOST=ADMIN_TEST_HOST)
|
||||
assert response.status_code == 200, response.content
|
||||
scoped = response.json()["active_crawls"][0]
|
||||
if scoped["crawl_output_size"] is not None or time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert scoped["total_snapshots"] == 1
|
||||
assert scoped["crawl_output_size"] == 9999
|
||||
assert scoped["cancelled_snapshots"] == 1
|
||||
|
||||
def test_live_progress_does_not_inspect_archive_files(self, client, admin_user, crawl, snapshot):
|
||||
import sys
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.config.constants import CONSTANTS
|
||||
|
||||
Crawl.objects.filter(pk=crawl.pk).update(status=Crawl.StatusChoices.STARTED)
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.STARTED)
|
||||
client.force_login(admin_user)
|
||||
# Warm imports/settings, then observe real calls without replacing any
|
||||
# filesystem API or the endpoint under test.
|
||||
assert client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST).status_code == 200
|
||||
filesystem_calls = []
|
||||
storage_roots = (CONSTANTS.ARCHIVE_DIR, CONSTANTS.PERSONAS_DIR)
|
||||
|
||||
def observe(frame, event, arg):
|
||||
if event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}:
|
||||
path = frame.f_locals.get("self")
|
||||
if isinstance(path, Path) and any(path.is_relative_to(root) for root in storage_roots):
|
||||
filesystem_calls.append(str(path))
|
||||
if event == "c_call" and arg in (os.scandir, os.listdir):
|
||||
filesystem_calls.append(arg.__name__)
|
||||
|
||||
previous_profile = sys.getprofile()
|
||||
sys.setprofile(observe)
|
||||
try:
|
||||
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
|
||||
finally:
|
||||
sys.setprofile(previous_profile)
|
||||
assert response.status_code == 200, response.content
|
||||
assert filesystem_calls == []
|
||||
|
||||
def test_live_progress_is_read_only_with_expired_machine(self, client, admin_user, crawl, snapshot):
|
||||
from datetime import timedelta
|
||||
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
import archivebox.machine.models as machine_models
|
||||
from archivebox.machine.models import Machine
|
||||
|
||||
machine = Machine.current()
|
||||
old_modified_at = timezone.now() - timedelta(days=8)
|
||||
Machine.objects.filter(pk=machine.pk).update(modified_at=old_modified_at)
|
||||
machine_models._CURRENT_MACHINE = None
|
||||
client.force_login(admin_user)
|
||||
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
|
||||
|
||||
assert response.status_code == 200, response.content
|
||||
writes = [query["sql"] for query in queries if query["sql"].lstrip().upper().startswith(("INSERT", "UPDATE", "DELETE", "REPLACE"))]
|
||||
assert writes == []
|
||||
machine.refresh_from_db()
|
||||
assert machine.modified_at == old_modified_at
|
||||
|
||||
def test_gallery_capture_fails_without_live_screencast(self, live_screenshot_command):
|
||||
command, env, variants = live_screenshot_command
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=75)
|
||||
elapsed = time.monotonic() - started
|
||||
assert result.returncode != 0, result.stdout
|
||||
assert "Live progress with a decoded screencast frame was not visible within 60 seconds" in result.stderr
|
||||
assert 60 <= elapsed < 75, (elapsed, result.stderr)
|
||||
assert not any(Path(variant["path"]).exists() for variant in variants)
|
||||
|
||||
def test_gallery_captures_real_sweeting_screencast(self, crawl, snapshot, live_screenshot_command):
|
||||
from struct import unpack
|
||||
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
from archivebox.services.runner import run_due_snapshot
|
||||
|
||||
config = {
|
||||
"PLUGINS": "chrome,chrome_screencast,title,wget,screenshot,dom,pdf",
|
||||
"PERMISSIONS": "public",
|
||||
"CHROME_HEADLESS": True,
|
||||
"CHROME_SANDBOX": False,
|
||||
"CHROME_ISOLATION": "snapshot",
|
||||
"CHROME_TIMEOUT": 90,
|
||||
"CHROME_PAGELOAD_TIMEOUT": 90,
|
||||
"CHROME_DELAY_AFTER_LOAD": 30,
|
||||
"TIMEOUT": 90,
|
||||
}
|
||||
Crawl.objects.filter(pk=crawl.pk).update(
|
||||
status=Crawl.StatusChoices.STARTED,
|
||||
retry_at=timezone.now(),
|
||||
urls=snapshot.url,
|
||||
config=config,
|
||||
)
|
||||
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now(), config=config)
|
||||
snapshot = Snapshot.objects.get(pk=snapshot.pk)
|
||||
errors = []
|
||||
|
||||
def run_snapshot():
|
||||
try:
|
||||
assert run_due_snapshot(snapshot, lock_seconds=120) is True
|
||||
except BaseException as err:
|
||||
errors.append(err)
|
||||
|
||||
runner = Thread(target=run_snapshot, name="archivebox-test-marketing-capture")
|
||||
command, env, variants = live_screenshot_command
|
||||
capture = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
runner.start()
|
||||
try:
|
||||
stdout, stderr = capture.communicate(timeout=75)
|
||||
assert capture.returncode == 0, stderr
|
||||
report = json.loads(stdout)
|
||||
assert report["checks"]["screencastImageLoaded"] is True
|
||||
captures = report["checks"]["liveProgressCaptures"]
|
||||
assert len(captures) == len(variants)
|
||||
for saved, variant in zip(captures, variants, strict=True):
|
||||
assert saved["ready"] is True, saved
|
||||
assert saved["screencastImageHasContent"] is True, saved
|
||||
assert saved["snapshotId"] == str(snapshot.pk).replace("-", "")
|
||||
assert saved["runningExtractors"] > 0
|
||||
assert saved["screencastImageUrl"].split("?")[0].endswith(f"/crawl/{crawl.pk}/files/chrome_screencast/latest.jpg")
|
||||
png = Path(variant["path"]).read_bytes()
|
||||
assert png[:8] == b"\x89PNG\r\n\x1a\n"
|
||||
assert unpack(">II", png[16:24]) == (variant["width"], variant["height"])
|
||||
Path(variants[0]["path"]).with_suffix(".json").write_text(stdout)
|
||||
finally:
|
||||
if capture.poll() is None:
|
||||
capture.terminate()
|
||||
capture.communicate(timeout=10)
|
||||
runner.join(timeout=120)
|
||||
assert not runner.is_alive()
|
||||
assert errors == []
|
||||
|
||||
def test_collection_summary_reports_stored_sizes(self, client, admin_user, snapshot):
|
||||
import time
|
||||
from django.core.cache import cache
|
||||
@@ -558,7 +841,7 @@ class TestLiveProgressView:
|
||||
try:
|
||||
os_process = psutil.Process(popen.pid)
|
||||
Process.objects.create(
|
||||
machine=Machine.current(refresh=True),
|
||||
machine=Machine.current(),
|
||||
process_type=Process.TypeChoices.ORCHESTRATOR,
|
||||
status=Process.StatusChoices.RUNNING,
|
||||
pid=popen.pid,
|
||||
|
||||
@@ -421,6 +421,12 @@ class TestPublicIndex:
|
||||
response = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
|
||||
|
||||
result_queries = [query["sql"].lower() for query in captured_queries if "core_archiveresult" in query["sql"].lower()]
|
||||
row_lookups = [
|
||||
query["sql"]
|
||||
for query in captured_queries
|
||||
if re.search(r'WHERE "(?:core_snapshot|crawls_crawl|auth_user)"\."id" =', query["sql"])
|
||||
]
|
||||
assert row_lookups == [], "Public rows must reuse the page's snapshot, crawl, and owner metadata"
|
||||
assert len(result_queries) == 1
|
||||
assert "output_files" in result_queries[0]
|
||||
assert "output_str" in result_queries[0]
|
||||
@@ -435,6 +441,102 @@ class TestPublicIndex:
|
||||
assert b'data-tooltip="singlefile"' in response.content
|
||||
assert b'title="singlefile"' not in response.content
|
||||
|
||||
@override_settings(PUBLIC_INDEX=True)
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_public_count_uses_permission_index_for_uniform_visibility(self, client, admin_user):
|
||||
import time
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.db import connection
|
||||
from django.db.backends.signals import connection_created
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
from archivebox.core.models import Snapshot
|
||||
from archivebox.crawls.models import Crawl
|
||||
|
||||
crawl = Crawl.objects.create(urls="https://public-count.example", created_by=admin_user, config={"PERMISSIONS": "public"})
|
||||
Snapshot.objects.bulk_create(
|
||||
[
|
||||
Snapshot(
|
||||
url=f"https://public-count.example/{index}",
|
||||
timestamp=str(1800000000 + index),
|
||||
crawl=crawl,
|
||||
config={"PERMISSIONS": "public"},
|
||||
status="sealed",
|
||||
)
|
||||
for index in range(125)
|
||||
],
|
||||
)
|
||||
if connection.vendor == "sqlite":
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("ANALYZE core_snapshot")
|
||||
|
||||
key = "progress-collection:public"
|
||||
cache.delete_many([key, f"{key}:refresh"])
|
||||
background_queries = []
|
||||
|
||||
def observe(execute, sql, params, many, context):
|
||||
background_queries.append((sql, params))
|
||||
return execute(sql, params, many, context)
|
||||
|
||||
def observe_background_connection(sender, connection, **kwargs):
|
||||
connection.execute_wrappers.append(observe)
|
||||
|
||||
connection_created.connect(observe_background_connection)
|
||||
try:
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
response = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert not any("COUNT(" in q["sql"] for q in queries), "Public page loads must not wait for a count"
|
||||
assert response.context["public_snapshot_count"] is None
|
||||
assert len(response.context["object_list"]) == 50
|
||||
assert response.context["page_obj"].has_next()
|
||||
assert b"of 125" not in response.content
|
||||
|
||||
deadline = time.monotonic() + 5
|
||||
while cache.get(key) is None and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert cache.get(key)["snapshots"] == 125
|
||||
|
||||
reloaded = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
|
||||
assert reloaded.context["public_snapshot_count"] == 125
|
||||
assert b"of 125" in reloaded.content
|
||||
last_page = client.get("/public/?page=3", HTTP_HOST=WEB_TEST_HOST)
|
||||
assert len(last_page.context["object_list"]) == 25
|
||||
assert not last_page.context["page_obj"].has_next()
|
||||
assert client.get("/public/?page=4", HTTP_HOST=WEB_TEST_HOST).status_code == 404
|
||||
|
||||
# The runner can add a page before the cached total refreshes.
|
||||
# Navigation follows actual rows, never the old cached page count.
|
||||
Snapshot.objects.bulk_create(
|
||||
[
|
||||
Snapshot(
|
||||
url=f"https://public-count.example/{index}",
|
||||
timestamp=str(1800000000 + index),
|
||||
crawl=crawl,
|
||||
config={"PERMISSIONS": "public"},
|
||||
status="sealed",
|
||||
)
|
||||
for index in range(125, 160)
|
||||
],
|
||||
)
|
||||
added_page = client.get("/public/?page=4", HTTP_HOST=WEB_TEST_HOST)
|
||||
assert added_page.status_code == 200
|
||||
assert len(added_page.context["object_list"]) == 10
|
||||
assert not added_page.context["page_obj"].has_next()
|
||||
assert added_page.context["public_snapshot_count"] == 160
|
||||
assert cache.get(key)["snapshots"] == 125
|
||||
|
||||
count_queries = [(sql, params) for sql, params in background_queries if "COUNT(" in sql and '"core_snapshot"' in sql]
|
||||
assert len(count_queries) == 1, "Reloads must reuse the cached total"
|
||||
if connection.vendor == "sqlite":
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("EXPLAIN QUERY PLAN " + count_queries[0][0], count_queries[0][1])
|
||||
plan = " ".join(str(row[-1]) for row in cursor.fetchall())
|
||||
assert "SEARCH core_snapshot USING INDEX" in plan and "permissions>" in plan, plan
|
||||
finally:
|
||||
connection_created.disconnect(observe_background_connection)
|
||||
|
||||
@override_settings(PUBLIC_INDEX=True)
|
||||
def test_upgraded_07_snapshot_icon_target_renders_at_extensionless_live_url(self, client, admin_user):
|
||||
from archivebox.core.models import ArchiveResult, Snapshot
|
||||
|
||||
@@ -11,6 +11,7 @@ import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from contextlib import ExitStack
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
@@ -1460,6 +1461,7 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep
|
||||
next_ownership_check = 0.0
|
||||
try:
|
||||
while follow:
|
||||
raise_if_shutdown_requested()
|
||||
now = time.monotonic()
|
||||
if keep_running is not None and now >= next_ownership_check:
|
||||
if not keep_running():
|
||||
@@ -1476,6 +1478,9 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep
|
||||
# Read ALL available lines from all files (not just one per iteration)
|
||||
for log_path, f in file_handles:
|
||||
while True:
|
||||
# A noisy worker may never reach EOF; shutdown must not
|
||||
# depend on returning to the outer poll loop.
|
||||
raise_if_shutdown_requested()
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break # No more lines available in this file
|
||||
@@ -1489,10 +1494,8 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep
|
||||
if not had_output:
|
||||
time.sleep(0.05)
|
||||
|
||||
except (KeyboardInterrupt, BrokenPipeError, OSError):
|
||||
except (KeyboardInterrupt, SystemExit, OSError):
|
||||
return "interrupted" # Let the caller handle the cleanup message
|
||||
except SystemExit:
|
||||
return "interrupted"
|
||||
finally:
|
||||
# Close all file handles
|
||||
for _, f in file_handles:
|
||||
@@ -1678,38 +1681,46 @@ def start_server_workers(
|
||||
config = get_config()
|
||||
shutdown_state = None
|
||||
tail_result = "stopped"
|
||||
try:
|
||||
supervisor = get_or_create_supervisord_process(daemonize=daemonize)
|
||||
workers, log_files, components = build_server_worker_plan(
|
||||
config=config,
|
||||
host=host,
|
||||
port=port,
|
||||
debug=debug,
|
||||
reload=reload,
|
||||
nothreading=nothreading,
|
||||
supervisor=supervisor,
|
||||
)
|
||||
component_list = format_runtime_components(components)
|
||||
if resumed_from_pid:
|
||||
print(
|
||||
"[yellow][*] Other newer archivebox process "
|
||||
f"(pid={resumed_from_pid}) exited, taking over {component_list} in this process again...[/yellow]",
|
||||
)
|
||||
else:
|
||||
print(f"[*] Starting {component_list} in this process (pid={os.getpid()})...")
|
||||
|
||||
print()
|
||||
sync_supervisord_workers(supervisor, workers, prune=True)
|
||||
print()
|
||||
|
||||
if daemonize:
|
||||
return None
|
||||
|
||||
from django.db import connections
|
||||
|
||||
connections.close_all()
|
||||
# Enter cooperative handling only once startup finishes, but retain it
|
||||
# through our cleanup finally: Popen.wait() has the same interruption risk
|
||||
# as poll(). ExitStack preserves the existing startup and daemon policies.
|
||||
with ExitStack() as shutdown_context:
|
||||
try:
|
||||
with foreground_shutdown_signals() as shutdown_state:
|
||||
supervisor = get_or_create_supervisord_process(daemonize=daemonize)
|
||||
workers, log_files, components = build_server_worker_plan(
|
||||
config=config,
|
||||
host=host,
|
||||
port=port,
|
||||
debug=debug,
|
||||
reload=reload,
|
||||
nothreading=nothreading,
|
||||
supervisor=supervisor,
|
||||
)
|
||||
component_list = format_runtime_components(components)
|
||||
if resumed_from_pid:
|
||||
print(
|
||||
"[yellow][*] Other newer archivebox process "
|
||||
f"(pid={resumed_from_pid}) exited, taking over {component_list} in this process again...[/yellow]",
|
||||
)
|
||||
else:
|
||||
print(f"[*] Starting {component_list} in this process (pid={os.getpid()})...")
|
||||
|
||||
print()
|
||||
sync_supervisord_workers(supervisor, workers, prune=True)
|
||||
print()
|
||||
|
||||
if daemonize:
|
||||
return None
|
||||
|
||||
from django.db import connections
|
||||
|
||||
connections.close_all()
|
||||
# Immediate exceptions can strand Popen's wait lock inside poll().
|
||||
# Use the shared cooperative contract in core.shutdown_util; the
|
||||
# tail checks intent between operations, including while draining
|
||||
# busy logs. No interruption checks run inside child cleanup.
|
||||
shutdown_state = shutdown_context.enter_context(foreground_shutdown_signals(raise_on_first_signal=False))
|
||||
try:
|
||||
# Tail worker logs while supervisord runs.
|
||||
sys.stdout.write("Tailing worker logs (Ctrl+C to stop)...\n\n")
|
||||
sys.stdout.flush()
|
||||
@@ -1719,26 +1730,24 @@ def start_server_workers(
|
||||
proc=_supervisord_proc, # Stop tailing when supervisord exits
|
||||
keep_running=keep_running,
|
||||
)
|
||||
except (KeyboardInterrupt, BrokenPipeError, OSError):
|
||||
if daemonize:
|
||||
raise
|
||||
if not shutdown_state or not shutdown_state.signal_name:
|
||||
print("\n[🛑] Got CTRL+C, stopping gracefully...")
|
||||
except SystemExit:
|
||||
if daemonize:
|
||||
raise
|
||||
except BaseException as e:
|
||||
if daemonize:
|
||||
raise
|
||||
STDERR.print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping gracefully...")
|
||||
finally:
|
||||
signal_shutdown_requested = bool(shutdown_state and shutdown_state.signal_name)
|
||||
if not daemonize and (signal_shutdown_requested or should_stop_supervisord is None or should_stop_supervisord()):
|
||||
# Ensure supervisord and all children are stopped only while this
|
||||
# foreground parent is still the active server parent. Standby
|
||||
# parents must not tear down a newer leader's services. If this
|
||||
# foreground parent itself received an OS shutdown signal, always
|
||||
# stop the supervisord child it owns; stop_own_supervisord_process()
|
||||
# does not target supervisord processes owned by other parents.
|
||||
stop_own_supervisord_process(record_exit=not signal_shutdown_requested)
|
||||
return tail_result
|
||||
except (KeyboardInterrupt, OSError):
|
||||
if not shutdown_state.signal_name:
|
||||
print("\n[🛑] Got CTRL+C, stopping gracefully...")
|
||||
except SystemExit:
|
||||
pass
|
||||
except BaseException as e:
|
||||
STDERR.print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping gracefully...")
|
||||
finally:
|
||||
if not daemonize:
|
||||
signal_shutdown_requested = bool(shutdown_state and shutdown_state.signal_name)
|
||||
stop_owned = signal_shutdown_requested or should_stop_supervisord is None or should_stop_supervisord()
|
||||
# The ownership callback can itself receive the first signal.
|
||||
signal_shutdown_requested = bool(shutdown_state and shutdown_state.signal_name)
|
||||
if stop_owned or signal_shutdown_requested:
|
||||
# A standby parent must never tear down the replacement
|
||||
# leader's services. Signals stop only our own Popen child,
|
||||
# even if leadership changed; they avoid DB exit writes.
|
||||
stop_own_supervisord_process(record_exit=not signal_shutdown_requested)
|
||||
# A signal can arrive during the final ownership/process-exit check. Do not
|
||||
# send the outer server loop back into standby or restart after shutdown.
|
||||
return "interrupted" if shutdown_state and shutdown_state.signal_name else tail_result
|
||||
+61
-188
@@ -118,10 +118,22 @@ trap cleanup EXIT INT TERM
|
||||
|
||||
mkdir -p "$DATA_DIR" "$OUTPUT_DIR" "$PUBLIC_OUTPUT_DIR"
|
||||
|
||||
PLUGIN_SCREENSHOT_PLAN="$CAPTURE_ROOT/plugin-screenshots.json"
|
||||
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" plugin-plan >"$PLUGIN_SCREENSHOT_PLAN"
|
||||
PLUGIN_ENABLED_CONFIG=()
|
||||
while IFS= read -r enabled_config; do
|
||||
[[ -z "$enabled_config" ]] || PLUGIN_ENABLED_CONFIG+=("$enabled_config")
|
||||
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json,sys; print("\n".join(f"{plugin.upper()}_ENABLED=True" for plugin in json.load(open(sys.argv[1]))["enabled"]))' \
|
||||
"$PLUGIN_SCREENSHOT_PLAN")
|
||||
|
||||
echo "[*] Initializing the ArchiveBox collection at $DATA_DIR"
|
||||
(
|
||||
cd "$DATA_DIR"
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox init --quick
|
||||
if [[ "${#PLUGIN_ENABLED_CONFIG[@]}" != "0" ]]; then
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox config --set "${PLUGIN_ENABLED_CONFIG[@]}"
|
||||
fi
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox install
|
||||
)
|
||||
|
||||
@@ -145,7 +157,7 @@ if [[ "$HAS_USEFUL_SNAPSHOT" == "0" ]]; then
|
||||
--overwrite \
|
||||
--tag=documentation,reference \
|
||||
--plugins=title,headers,wget,screenshot,pdf,dom,readability,htmltotext,hashes,dns \
|
||||
https://example.com https://archivebox.io
|
||||
https://docs.sweeting.me/s/cookie-dilemma https://archivebox.io
|
||||
)
|
||||
fi
|
||||
|
||||
@@ -209,11 +221,7 @@ PYPORT
|
||||
)"
|
||||
(
|
||||
cd "$DATA_DIR"
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox config --set \
|
||||
OPENCODE_ENABLED=True "OPENCODE_PORT=$OPENCODE_PORT" TLSNOTARY_ENABLED=True \
|
||||
HASHES_ENABLED=True OPENTIMESTAMPS_ENABLED=True
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox install opencode --binproviders=env,pnpm
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox install opentimestamps
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox config --set "OPENCODE_PORT=$OPENCODE_PORT"
|
||||
)
|
||||
|
||||
echo "[*] Starting ArchiveBox on port $PORT"
|
||||
@@ -262,43 +270,16 @@ VIEWS=(
|
||||
"Login|$ADMIN_BASE_URL/admin/login/|/admin/login/|archivebox/templates/admin/login.html"
|
||||
"Public snapshot list|$PUBLIC_BASE_URL/public/|/public/|archivebox/core/views.py"
|
||||
)
|
||||
REQUIRED_TEMPLATE_PLUGINS=(
|
||||
accessibility
|
||||
chrome
|
||||
consolelog
|
||||
defuddle
|
||||
dns
|
||||
hashes
|
||||
headers
|
||||
htmltotext
|
||||
liteparse
|
||||
opendataloader
|
||||
parse_dom_outlinks
|
||||
parse_html_urls
|
||||
parse_jsonl_urls
|
||||
parse_netscape_urls
|
||||
parse_rss_urls
|
||||
parse_txt_urls
|
||||
redirects
|
||||
seo
|
||||
sslcerts
|
||||
title
|
||||
trafilatura
|
||||
)
|
||||
DISCOVERED_TEMPLATE_PLUGINS=()
|
||||
|
||||
add_supplementary_template_capture() {
|
||||
local plugin_name="$1"
|
||||
local plugin_names="$1"
|
||||
local source_url="$2"
|
||||
local capture_plugins="$plugin_name"
|
||||
local capture_log="$CAPTURE_ROOT/${plugin_name}-supplementary-capture.log"
|
||||
local supplementary_plugins=("$plugin_name")
|
||||
|
||||
if [[ "$plugin_name" == "opendataloader" ]]; then
|
||||
capture_plugins="wget"
|
||||
supplementary_plugins=(liteparse opendataloader)
|
||||
fi
|
||||
echo "[*] Capturing real $plugin_name source for template coverage"
|
||||
local prepare_plugins="$3"
|
||||
local capture_plugins="${prepare_plugins:-$plugin_names}"
|
||||
local capture_log="$CAPTURE_ROOT/${plugin_names}-supplementary-capture.log"
|
||||
local supplementary_plugins=()
|
||||
IFS=',' read -r -a supplementary_plugins <<<"$plugin_names"
|
||||
echo "[*] Capturing real $plugin_names source for template coverage"
|
||||
if ! (
|
||||
cd "$DATA_DIR"
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox add \
|
||||
@@ -308,7 +289,7 @@ add_supplementary_template_capture() {
|
||||
--plugins="$capture_plugins" \
|
||||
"$source_url"
|
||||
) >"$capture_log" 2>&1; then
|
||||
echo "[!] Supplementary $plugin_name capture failed" >&2
|
||||
echo "[!] Supplementary $plugin_names capture failed" >&2
|
||||
tail -100 "$capture_log" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -321,37 +302,37 @@ add_supplementary_template_capture() {
|
||||
'import os; from urllib.parse import urlsplit; from archivebox.core.models import Snapshot; from archivebox.core.routes_util import build_snapshot_detail_url; snapshot=Snapshot.objects.filter(url=os.environ["UI_SCREENSHOT_SOURCE_URL"]).order_by("-bookmarked_at").first(); url=build_snapshot_detail_url(snapshot.archive_path_from_db) if snapshot else ""; print(f"{snapshot.id}\t{url}\t{urlsplit(url).path}" if snapshot else "")'
|
||||
) | tail -1)"
|
||||
if [[ -z "$snapshot_record" ]]; then
|
||||
echo "[!] Supplementary $plugin_name capture did not create a snapshot" >&2
|
||||
echo "[!] Supplementary $plugin_names capture did not create a snapshot" >&2
|
||||
tail -100 "$capture_log" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local snapshot_id snapshot_view_url snapshot_view_path
|
||||
IFS=$'\t' read -r snapshot_id snapshot_view_url snapshot_view_path <<<"$snapshot_record"
|
||||
if [[ "$plugin_name" == "opendataloader" ]]; then
|
||||
if [[ -n "$prepare_plugins" ]]; then
|
||||
if ! (
|
||||
cd "$DATA_DIR"
|
||||
LITEPARSE_ENABLED=True OPENDATALOADER_ENABLED=True uv run --no-cache --project "$REPO_DIR" archivebox extract \
|
||||
--plugins=liteparse,opendataloader \
|
||||
uv run --no-cache --project "$REPO_DIR" archivebox extract \
|
||||
--plugins="$plugin_names" \
|
||||
"$snapshot_id"
|
||||
) >>"$capture_log" 2>&1; then
|
||||
echo "[!] LiteParse/OpenDataLoader extraction failed for the captured wget PDF" >&2
|
||||
echo "[!] $plugin_names extraction failed for the prepared snapshot" >&2
|
||||
tail -100 "$capture_log" >&2
|
||||
exit 1
|
||||
fi
|
||||
stop_background_runner
|
||||
fi
|
||||
|
||||
local discovery_report="$CAPTURE_ROOT/${plugin_name}-output-discovery.json"
|
||||
local discovery_report="$CAPTURE_ROOT/${plugin_names}-output-discovery.json"
|
||||
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
|
||||
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
|
||||
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
|
||||
SCREENSHOT_SNAPSHOT_HEADER=collapsed \
|
||||
SCREENSHOT_WIDTH=1600 \
|
||||
SCREENSHOT_HEIGHT=1000 \
|
||||
node "$REPO_DIR/bin/take_screenshot.js" "$snapshot_view_url" "$CAPTURE_ROOT/${plugin_name}-output-discovery.png" >"$discovery_report"
|
||||
node "$REPO_DIR/bin/take_screenshot.js" "$snapshot_view_url" "$CAPTURE_ROOT/${plugin_names}-output-discovery.png" >"$discovery_report"
|
||||
|
||||
local supplementary_plugin discovered_preview_url plugin_already_discovered discovered_plugin
|
||||
local supplementary_plugin discovered_preview_url capture_mode
|
||||
for supplementary_plugin in "${supplementary_plugins[@]}"; do
|
||||
discovered_preview_url="$(
|
||||
UI_SCREENSHOT_DISCOVERY_REPORT="$discovery_report" UI_SCREENSHOT_EXPECT_PLUGIN="$supplementary_plugin" \
|
||||
@@ -365,17 +346,10 @@ add_supplementary_template_capture() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
plugin_already_discovered=0
|
||||
for discovered_plugin in "${DISCOVERED_TEMPLATE_PLUGINS[@]}"; do
|
||||
if [[ "$discovered_plugin" == "$supplementary_plugin" ]]; then
|
||||
plugin_already_discovered=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$plugin_already_discovered" == "0" ]]; then
|
||||
DISCOVERED_TEMPLATE_PLUGINS+=("$supplementary_plugin")
|
||||
VIEWS+=("Snapshot View ($supplementary_plugin)|$snapshot_view_url#$supplementary_plugin|$snapshot_view_path|archivebox/templates/core/snapshot.html")
|
||||
fi
|
||||
capture_mode="$(uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json,sys; recipe=json.load(open(sys.argv[1]))["overrides"][sys.argv[2]]; text=recipe.get("wait_for_text", ""); print("wait-preview-text:" + text if text else "")' \
|
||||
"$PLUGIN_SCREENSHOT_PLAN" "$supplementary_plugin")"
|
||||
VIEWS+=("Snapshot View ($supplementary_plugin)|$snapshot_view_url#$supplementary_plugin|$snapshot_view_path|archivebox/templates/core/snapshot.html|$capture_mode")
|
||||
done
|
||||
}
|
||||
|
||||
@@ -385,49 +359,19 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
|
||||
IFS='|' read -r name url expected_path source capture_mode <<<"$view"
|
||||
capture_index=$((capture_index + 1))
|
||||
slug="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//; s/-$//')"
|
||||
expected_plugin=""
|
||||
if [[ "$name" == "Snapshot View ("*")" && "$capture_mode" != "live-progress" && "$capture_mode" != "snapshot-collapsed" ]]; then
|
||||
expected_plugin="${name#Snapshot View (}"
|
||||
expected_plugin="${expected_plugin%)}"
|
||||
fi
|
||||
|
||||
echo "[*] $name: $url"
|
||||
if [[ "$capture_mode" == "snapshot-collapsed" ]]; then
|
||||
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
|
||||
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
|
||||
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
|
||||
SCREENSHOT_SNAPSHOT_HEADER=collapsed \
|
||||
SCREENSHOT_WIDTH=1600 \
|
||||
SCREENSHOT_HEIGHT=1000 \
|
||||
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$CAPTURE_ROOT/snapshot-header-state.png" >/dev/null
|
||||
fi
|
||||
snapshot_header=expanded
|
||||
[[ "$name" != "Snapshot View ("* ]] || snapshot_header=collapsed
|
||||
[[ "$capture_mode" != "snapshot-collapsed" ]] || snapshot_header=collapsed
|
||||
wait_for_text=""
|
||||
[[ "$capture_mode" != wait-text:* ]] || wait_for_text="${capture_mode#wait-text:}"
|
||||
preview_wait_for_text=""
|
||||
[[ "$capture_mode" != wait-preview-text:* ]] || preview_wait_for_text="${capture_mode#wait-preview-text:}"
|
||||
view_timing_report=""
|
||||
while IFS='|' read -r profile viewport_width viewport_height; do
|
||||
while IFS='|' read -r profile _viewport_width _viewport_height; do
|
||||
filename="$slug-$profile.png"
|
||||
capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/$profile"
|
||||
timing_report_path="$view_timing_report"
|
||||
capture_env=(
|
||||
"RESOLUTION=$viewport_width,$viewport_height"
|
||||
"CHROME_RESOLUTION=$viewport_width,$viewport_height"
|
||||
"PERSONAS_DIR=$PERSONAS_DIR"
|
||||
"ACTIVE_PERSONA=$ACTIVE_PERSONA"
|
||||
"SCREENSHOT_RESOLUTION=$viewport_width,$viewport_height"
|
||||
"SCREENSHOT_TIMEOUT=120"
|
||||
"CHROME_WAIT_FOR=load"
|
||||
"SCREENSHOT_COLLAPSE_FILTERS=1"
|
||||
)
|
||||
if [[ "$capture_mode" == wait-replay:* ]]; then
|
||||
capture_env+=(
|
||||
"SCREENSHOT_WAIT_FOR_TEXT=${capture_mode#wait-replay:}"
|
||||
"SCREENSHOT_WAIT_FOR_FRAME_URL=/replay/w/"
|
||||
)
|
||||
elif [[ "$capture_mode" == wait-text:* ]]; then
|
||||
capture_env+=(
|
||||
"CHROME_WAIT_FOR=domcontentloaded"
|
||||
"SCREENSHOT_WAIT_FOR_TEXT=${capture_mode#wait-text:}"
|
||||
)
|
||||
fi
|
||||
|
||||
screenshot_path=""
|
||||
if [[ "$capture_mode" == "setup-wizard" ]]; then
|
||||
screenshot_path="$capture_dir/screenshot.png"
|
||||
@@ -521,12 +465,13 @@ PY
|
||||
SCREENSHOT_WIDTH=1600 \
|
||||
SCREENSHOT_HEIGHT=1000 \
|
||||
SCREENSHOT_VARIANTS_JSON="$live_variants" \
|
||||
SCREENSHOT_REQUIRE_LIVE_PROGRESS=1 \
|
||||
SCREENSHOT_SNAPSHOT_HEADER=expanded \
|
||||
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
|
||||
view_timing_report="$capture_dir/report.json"
|
||||
timing_report_path="$view_timing_report"
|
||||
fi
|
||||
elif [[ -n "$expected_plugin" ]]; then
|
||||
else
|
||||
screenshot_path="$capture_dir/screenshot.png"
|
||||
if [[ "$profile" == "desktop" ]]; then
|
||||
mkdir -p \
|
||||
@@ -538,58 +483,16 @@ PY
|
||||
"$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop/screenshot.png" \
|
||||
"$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet/screenshot.png" \
|
||||
"$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile/screenshot.png")"
|
||||
env "${capture_env[@]}" \
|
||||
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
|
||||
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
|
||||
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
|
||||
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
|
||||
SCREENSHOT_WIDTH=1600 \
|
||||
SCREENSHOT_HEIGHT=1000 \
|
||||
SCREENSHOT_VARIANTS_JSON="$output_variants" \
|
||||
SCREENSHOT_COLLAPSE_FILTERS=1 \
|
||||
SCREENSHOT_SNAPSHOT_HEADER=expanded \
|
||||
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
|
||||
view_timing_report="$capture_dir/report.json"
|
||||
timing_report_path="$view_timing_report"
|
||||
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
|
||||
"$capture_dir/report.json" "$expected_path"
|
||||
fi
|
||||
elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then
|
||||
capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log"
|
||||
if ! env "${capture_env[@]}" uv run --no-cache --project "$REPO_DIR" abx-dl dl \
|
||||
--plugins=screenshot \
|
||||
--timeout=120 \
|
||||
--dir "$capture_dir" \
|
||||
"$url" >"$capture_log" 2>&1; then
|
||||
echo "[!] abx-dl failed while capturing $profile $url" >&2
|
||||
tail -100 "$capture_log" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
screenshot_path="$(find "$capture_dir" -type f -path '*/screenshot/screenshot.png' -print -quit)"
|
||||
screenshot_metadata_path="$(find "$capture_dir" -type f -path '*/screenshot/screenshot.json' -print -quit)"
|
||||
if [[ -z "$screenshot_metadata_path" || ! -s "$screenshot_metadata_path" ]]; then
|
||||
echo "[!] Screenshot navigation metadata is missing for $profile $url" >&2
|
||||
tail -100 "$capture_log" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
|
||||
"$screenshot_metadata_path" "$expected_path"
|
||||
else
|
||||
screenshot_path="$capture_dir/screenshot.png"
|
||||
if [[ "$profile" == "tablet" ]]; then
|
||||
mobile_capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile"
|
||||
mkdir -p "$capture_dir" "$mobile_capture_dir"
|
||||
responsive_variants="$(printf \
|
||||
'[{"path":"%s","width":1024,"height":1366},{"path":"%s","width":390,"height":844}]' \
|
||||
"$capture_dir/screenshot.png" \
|
||||
"$mobile_capture_dir/screenshot.png")"
|
||||
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
|
||||
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
|
||||
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
|
||||
SCREENSHOT_WIDTH=1024 \
|
||||
SCREENSHOT_HEIGHT=1366 \
|
||||
SCREENSHOT_VARIANTS_JSON="$responsive_variants" \
|
||||
SCREENSHOT_COLLAPSE_FILTERS=1 \
|
||||
SCREENSHOT_SNAPSHOT_HEADER="$snapshot_header" \
|
||||
SCREENSHOT_WAIT_FOR_TEXT="$wait_for_text" \
|
||||
SCREENSHOT_PREVIEW_WAIT_FOR_TEXT="$preview_wait_for_text" \
|
||||
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
|
||||
view_timing_report="$capture_dir/report.json"
|
||||
timing_report_path="$view_timing_report"
|
||||
@@ -598,8 +501,7 @@ PY
|
||||
fi
|
||||
fi
|
||||
if [[ -z "$screenshot_path" || ! -s "$screenshot_path" ]]; then
|
||||
echo "[!] Screenshot plugin did not produce a $profile PNG for $url" >&2
|
||||
[[ -n "${capture_log:-}" && -f "$capture_log" ]] && tail -100 "$capture_log" >&2
|
||||
echo "[!] Screenshot capture did not produce a $profile PNG for $url" >&2
|
||||
exit 1
|
||||
fi
|
||||
cp "$screenshot_path" "$OUTPUT_DIR/$filename"
|
||||
@@ -649,19 +551,21 @@ PY
|
||||
) >"$CAPTURE_ROOT/sweeting-live-capture.log" 2>&1 &
|
||||
ARCHIVE_PID=$!
|
||||
|
||||
# Open the detail page once archiving starts, so its initial state and
|
||||
# the live monitor both show the capture in progress.
|
||||
LIVE_SNAPSHOT_VIEW_URL=""
|
||||
for _attempt in $(seq 1 120); do
|
||||
LIVE_SNAPSHOT_RECORD="$( (
|
||||
cd "$DATA_DIR"
|
||||
UI_SCREENSHOT_CAPTURE_STARTED_AT="$SWEETING_CAPTURE_STARTED_AT" uv run --no-cache --project "$REPO_DIR" archivebox manage shell --no-imports -c \
|
||||
'import os; from urllib.parse import urlsplit; from django.utils.dateparse import parse_datetime; from archivebox.core.models import Snapshot; from archivebox.core.routes_util import build_snapshot_detail_url; started_at=parse_datetime(os.environ["UI_SCREENSHOT_CAPTURE_STARTED_AT"]); snapshot=Snapshot.objects.filter(url__startswith="https://sweeting.me",bookmarked_at__gte=started_at).order_by("-bookmarked_at").first(); url=build_snapshot_detail_url(snapshot.archive_path_from_db) if snapshot else ""; print(f"{url}\t{urlsplit(url).path}" if snapshot else "")'
|
||||
'import os; from urllib.parse import urlsplit; from django.utils.dateparse import parse_datetime; from archivebox.core.models import Snapshot; from archivebox.core.routes_util import build_snapshot_detail_url; started_at=parse_datetime(os.environ["UI_SCREENSHOT_CAPTURE_STARTED_AT"]); snapshot=Snapshot.objects.filter(url__startswith="https://sweeting.me",bookmarked_at__gte=started_at,status="started").order_by("-bookmarked_at").first(); url=build_snapshot_detail_url(snapshot.archive_path_from_db) if snapshot else ""; print(f"{url}\t{urlsplit(url).path}" if snapshot else "")'
|
||||
) | tail -1)"
|
||||
if [[ -n "$LIVE_SNAPSHOT_RECORD" ]]; then
|
||||
IFS=$'\t' read -r LIVE_SNAPSHOT_VIEW_URL LIVE_SNAPSHOT_VIEW_PATH <<<"$LIVE_SNAPSHOT_RECORD"
|
||||
break
|
||||
fi
|
||||
if ! kill -0 "$ARCHIVE_PID" 2>/dev/null; then
|
||||
echo "[!] Sweeting.me capture exited before creating a snapshot" >&2
|
||||
echo "[!] Sweeting.me capture exited before an active snapshot could be opened" >&2
|
||||
tail -100 "$CAPTURE_ROOT/sweeting-live-capture.log" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -748,54 +652,23 @@ PY
|
||||
SCREENSHOT_WIDTH=1600 \
|
||||
SCREENSHOT_HEIGHT=1000 \
|
||||
node "$REPO_DIR/bin/take_screenshot.js" "$LIVE_SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT"
|
||||
SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); print("\n".join("{}\t{}\t{}".format(output["plugin"], output["outputPath"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"]))')"
|
||||
SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_PLAN="$PLUGIN_SCREENSHOT_PLAN" UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); overrides=json.load(open(os.environ["UI_SCREENSHOT_PLAN"]))["overrides"]; print("\n".join("{}\t{}\t{}".format(output["plugin"], output["outputPath"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"] if output["plugin"] not in overrides))')"
|
||||
if [[ -z "$SNAPSHOT_OUTPUT_PLUGINS" ]]; then
|
||||
echo "[!] The Sweeting.me snapshot detail page exposed no selectable outputs" >&2
|
||||
exit 1
|
||||
fi
|
||||
for required_plugin in tlsnotary opentimestamps; do
|
||||
if ! printf '%s\n' "$SNAPSHOT_OUTPUT_PLUGINS" | cut -f1 | grep -qx "$required_plugin"; then
|
||||
echo "[!] Live capture did not expose the required $required_plugin output" >&2
|
||||
tail -200 "$CAPTURE_ROOT/sweeting-live-capture.log" >&2
|
||||
cat "$SNAPSHOT_DISCOVERY_REPORT" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
while IFS=$'\t' read -r plugin_name output_path output_capture_mode; do
|
||||
[[ -z "$plugin_name" ]] && continue
|
||||
DISCOVERED_TEMPLATE_PLUGINS+=("$plugin_name")
|
||||
VIEWS+=("Snapshot View ($plugin_name)|$LIVE_SNAPSHOT_VIEW_URL#$output_path|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|$output_capture_mode")
|
||||
done <<<"$SNAPSHOT_OUTPUT_PLUGINS"
|
||||
add_supplementary_template_capture \
|
||||
parse_jsonl_urls \
|
||||
https://raw.githubusercontent.com/noho/dayu-agent/f4343e490dd096e81a2f4e715c5592a10e126a26/utils/web_ci_urls.jsonl
|
||||
add_supplementary_template_capture \
|
||||
parse_rss_urls \
|
||||
https://github.com/ArchiveBox/ArchiveBox/releases.atom
|
||||
add_supplementary_template_capture \
|
||||
parse_netscape_urls \
|
||||
https://gist.githubusercontent.com/jgarber623/cdc8e2fa1cbcb6889872/raw/2081aac716fd9aeb2dc443cd7cace57b8e830c57/delicious.html
|
||||
add_supplementary_template_capture \
|
||||
opendataloader \
|
||||
https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf
|
||||
while IFS='|' read -r example_plugins example_url example_prepare; do
|
||||
[[ -z "$example_plugins" ]] && continue
|
||||
add_supplementary_template_capture "$example_plugins" "$example_url" "$example_prepare"
|
||||
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
|
||||
'import json,sys; plan=json.load(open(sys.argv[1])); print("\n".join("|".join((",".join(example["plugins"]), example["url"], ",".join(example["prepare_plugins"]))) for example in plan["examples"]))' \
|
||||
"$PLUGIN_SCREENSHOT_PLAN")
|
||||
|
||||
missing_template_plugins=()
|
||||
for required_plugin in "${REQUIRED_TEMPLATE_PLUGINS[@]}"; do
|
||||
plugin_was_discovered=0
|
||||
for discovered_plugin in "${DISCOVERED_TEMPLATE_PLUGINS[@]}"; do
|
||||
if [[ "$discovered_plugin" == "$required_plugin" ]]; then
|
||||
plugin_was_discovered=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
[[ "$plugin_was_discovered" == "0" ]] && missing_template_plugins+=("$required_plugin")
|
||||
done
|
||||
if [[ "$MAX_VIEWS" == "0" && "${#missing_template_plugins[@]}" != "0" ]]; then
|
||||
echo "[!] Real snapshot UI discovery is missing template coverage for: ${missing_template_plugins[*]}" >&2
|
||||
cat "$SNAPSHOT_DISCOVERY_REPORT" >&2
|
||||
exit 1
|
||||
fi
|
||||
VIEWS+=("Snapshot View (header collapsed)|$LIVE_SNAPSHOT_VIEW_URL|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|snapshot-collapsed")
|
||||
fi
|
||||
if [[ "$MAX_VIEWS" != "0" && "$capture_index" -ge "$MAX_VIEWS" ]]; then
|
||||
|
||||
@@ -32,7 +32,6 @@ REQUIRED_VIEW_NAMES = {
|
||||
"Snapshot admin detail",
|
||||
"Snapshot View (capture in progress)",
|
||||
"Snapshot View (header collapsed)",
|
||||
"Snapshot View (opentimestamps)",
|
||||
"Snapshot files",
|
||||
"Archive results",
|
||||
"Archive result detail",
|
||||
@@ -61,6 +60,41 @@ REQUIRED_VIEW_NAMES = {
|
||||
}
|
||||
|
||||
|
||||
def plugin_screenshot_plan() -> dict:
|
||||
"""Discover default coverage and plugin-owned examples from the installed package."""
|
||||
from abx_plugins import get_plugins_dir
|
||||
|
||||
required = []
|
||||
examples = {}
|
||||
overrides = {}
|
||||
enabled_plugins = []
|
||||
for plugin_dir in sorted(get_plugins_dir().iterdir()):
|
||||
config_path = plugin_dir / "config.json"
|
||||
if not config_path.is_file():
|
||||
continue
|
||||
config = json.loads(config_path.read_text())
|
||||
recipe = json.loads((plugin_dir / config["screenshot"]).read_text()) if config.get("screenshot") else {}
|
||||
plugin = plugin_dir.name
|
||||
enabled = config.get("properties", {}).get(f"{plugin.upper()}_ENABLED", {}).get("default", False)
|
||||
if recipe.get("enabled") or recipe.get("url"):
|
||||
enabled_plugins.append(plugin)
|
||||
enabled = True
|
||||
if recipe.get("view"):
|
||||
required.append(recipe["view"])
|
||||
elif enabled and (
|
||||
recipe.get("url")
|
||||
or (plugin_dir / "templates/full.html").is_file()
|
||||
or ((plugin_dir / "templates/card.html").is_file() and not config.get("card_hidden"))
|
||||
):
|
||||
required.append(f"Snapshot View ({plugin})")
|
||||
if recipe.get("url"):
|
||||
overrides[plugin] = recipe
|
||||
key = (recipe["url"], tuple(recipe.get("prepare_plugins", [])))
|
||||
example = examples.setdefault(key, {"url": key[0], "prepare_plugins": list(key[1]), "plugins": []})
|
||||
example["plugins"].append(plugin)
|
||||
return {"required": required, "examples": list(examples.values()), "overrides": overrides, "enabled": enabled_plugins}
|
||||
|
||||
|
||||
def build_provenance() -> dict[str, str]:
|
||||
version = tomllib.loads((REPO_DIR / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"]
|
||||
revision = os.environ.get("GITHUB_SHA", "").strip()
|
||||
@@ -213,19 +247,9 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
|
||||
|
||||
if not allow_partial:
|
||||
captured_names = set(captures_by_name)
|
||||
missing = sorted(REQUIRED_VIEW_NAMES - captured_names)
|
||||
missing = sorted((REQUIRED_VIEW_NAMES | set(plugin_screenshot_plan()["required"])) - captured_names)
|
||||
if missing:
|
||||
raise SystemExit(f"required UI screenshot coverage is missing: {', '.join(missing)}")
|
||||
snapshot_output_views = [
|
||||
name
|
||||
for name in captured_names
|
||||
if name.startswith("Snapshot View (")
|
||||
and name not in {"Snapshot View (capture in progress)", "Snapshot View (header collapsed)"}
|
||||
]
|
||||
if len(snapshot_output_views) < 20:
|
||||
raise SystemExit(
|
||||
f"expected at least 20 rendered Sweeting.me snapshot outputs, got {len(snapshot_output_views)}",
|
||||
)
|
||||
markdown_sections = []
|
||||
html_sections = []
|
||||
for capture in grouped_captures:
|
||||
@@ -276,7 +300,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
|
||||
),
|
||||
)
|
||||
html_sections.append(
|
||||
f"<article><h2>{html.escape(capture['name'])}</h2>"
|
||||
f'<article id="{re.sub(r"[^a-z0-9]+", "-", capture["name"].lower()).strip("-")}"><h2>{html.escape(capture["name"])}</h2>'
|
||||
f'<p><a href="{html.escape(capture["url"])}"><code>{html.escape(route)}</code></a> · '
|
||||
f'<a href="{html.escape(source_url)}">View code</a>{html.escape(ttfb_text)}</p>'
|
||||
f'<div class="shots">{"".join(html_figures)}</div></article>',
|
||||
@@ -345,6 +369,12 @@ def refresh_gallery(html_path: Path) -> None:
|
||||
if not header or not sections:
|
||||
raise SystemExit("Published screenshot gallery is missing its capture header or content")
|
||||
content = sections[1]
|
||||
content = re.sub(
|
||||
r"<article><h2>(.*?)</h2>",
|
||||
lambda match: f'<article id="{re.sub(r"[^a-z0-9]+", "-", html.unescape(match[1]).lower()).strip("-")}"><h2>{match[1]}</h2>',
|
||||
content,
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
for profile in CAPTURE_PROFILES:
|
||||
# Galleries published before the switcher used classes alone.
|
||||
content = content.replace(
|
||||
@@ -355,6 +385,9 @@ def refresh_gallery(html_path: Path) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if sys.argv[1:] == ["plugin-plan"]:
|
||||
print(json.dumps(plugin_screenshot_plan()))
|
||||
return
|
||||
if len(sys.argv) == 3 and sys.argv[1] == "refresh":
|
||||
refresh_gallery(Path(sys.argv[2]))
|
||||
return
|
||||
|
||||
+156
-10
@@ -26,6 +26,9 @@ Environment:
|
||||
SCREENSHOT_FULL_PAGE Set to 1 to capture the full page, defaults to viewport only
|
||||
SCREENSHOT_SCROLL_SELECTOR Scroll this selector into view before capture
|
||||
SCREENSHOT_WAIT_SELECTOR Wait for this selector before capture
|
||||
SCREENSHOT_WAIT_FOR_TEXT Wait for this visible page text before capture
|
||||
SCREENSHOT_PREVIEW_WAIT_FOR_TEXT Wait for this text in the selected snapshot preview iframe
|
||||
SCREENSHOT_REQUIRE_LIVE_PROGRESS Require active snapshot progress and a nonblank decoded screencast within 60s
|
||||
SCREENSHOT_CLICK_SELECTOR Click this selector before capture
|
||||
SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR Wait for this selector after clicking
|
||||
SCREENSHOT_HOST_RESOLVER_RULES Chrome host resolver rules
|
||||
@@ -61,7 +64,7 @@ function addLaunchArg(args, arg) {
|
||||
if (!args.includes(arg)) args.push(arg);
|
||||
}
|
||||
|
||||
async function waitForVisibleCardPreviews(page) {
|
||||
async function waitForVisibleCardPreviews(page, timeout = 45000) {
|
||||
// The page load event does not wait for nested lazy card images. Capture their
|
||||
// decoded pixels, without forcing hidden alternatives or offscreen cards to load.
|
||||
const cardHandles = await page.$$('iframe[title$=" card"]');
|
||||
@@ -106,10 +109,114 @@ async function waitForVisibleCardPreviews(page) {
|
||||
if (!images.every((img) => img.complete && img.naturalWidth > 0)) return false;
|
||||
await Promise.all(images.map((img) => img.decode()));
|
||||
return true;
|
||||
}, {timeout: 45000}, expectedUrl);
|
||||
}, {timeout}, expectedUrl);
|
||||
}));
|
||||
}
|
||||
|
||||
async function liveProgressState(readyOnly = false, rejectedFrameUrl = '') {
|
||||
const visible = (element, inViewport = true) => {
|
||||
if (!element || !element.checkVisibility({checkOpacity: true, checkVisibilityCSS: true})) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (rect.width <= 0 || rect.height <= 0) return false;
|
||||
if (!inViewport) return true;
|
||||
const left = Math.max(0, rect.left);
|
||||
const right = Math.min(innerWidth, rect.right);
|
||||
const top = Math.max(0, rect.top);
|
||||
const bottom = Math.min(innerHeight, rect.bottom);
|
||||
return right > left && bottom > top
|
||||
&& element.contains(document.elementFromPoint((left + right) / 2, (top + bottom) / 2));
|
||||
};
|
||||
const monitor = document.querySelector('#progress-monitor[data-progress-scope="snapshot"]');
|
||||
const panel = monitor?.querySelector('.screencast-panel.visible[data-mode="frame"]');
|
||||
const img = panel?.querySelector('.screencast-frame img');
|
||||
const src = img?.currentSrc || '';
|
||||
const snapshotId = monitor
|
||||
? new URL(monitor.dataset.progressEndpoint, location.href).searchParams.get('snapshot_id')
|
||||
: '';
|
||||
const snapshot = [...(monitor?.querySelectorAll('.snapshot-item') || [])]
|
||||
.find((item) => item.dataset.snapshotId?.replaceAll('-', '') === snapshotId?.replaceAll('-', ''));
|
||||
const state = {
|
||||
url: location.href,
|
||||
title: document.title,
|
||||
progressMonitorVisible: visible(monitor),
|
||||
progressDetailsVisible: visible(monitor?.querySelector('.progress-content')),
|
||||
snapshotId,
|
||||
snapshotStarted: Boolean(snapshot?.querySelector('.status-badge.started')),
|
||||
runningExtractors: [...(snapshot?.querySelectorAll('.extractor-badge.started') || [])]
|
||||
.filter((element) => visible(element, false)).length,
|
||||
screencastVisible: visible(panel) && visible(img),
|
||||
screencastPlaceholderVisible: visible(monitor?.querySelector('.screencast-placeholder')),
|
||||
screencastImageLoaded: Boolean(img?.complete && img.naturalWidth > 0 && img.naturalHeight > 0),
|
||||
screencastImageSize: img ? `${img.naturalWidth}x${img.naturalHeight}` : '',
|
||||
screencastImageUrl: src,
|
||||
};
|
||||
state.ready = state.progressMonitorVisible && state.progressDetailsVisible
|
||||
&& state.snapshotStarted && state.runningExtractors > 0
|
||||
&& state.screencastVisible && !state.screencastPlaceholderVisible && state.screencastImageLoaded
|
||||
&& /\/api\/v1\/crawls\/crawl\/[^/]+\/files\/chrome_screencast\/latest\.jpg$/.test(new URL(src || 'about:blank').pathname);
|
||||
if (!readyOnly) return state;
|
||||
if (!state.ready || src === rejectedFrameUrl) return false;
|
||||
try {
|
||||
await img.decode();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return img.isConnected && img.currentSrc === src && visible(img) ? state : false;
|
||||
}
|
||||
|
||||
async function waitForLiveProgress(page, deadline, screencastResponses) {
|
||||
let rejectedFrameUrl = '';
|
||||
try {
|
||||
while (Date.now() < deadline) {
|
||||
const handle = await page.waitForFunction(liveProgressState, {timeout: Math.max(1, deadline - Date.now()), polling: 100}, true, rejectedFrameUrl);
|
||||
const state = await handle.jsonValue();
|
||||
await handle.dispose();
|
||||
// Chrome's initial frame can be a fully decoded, solid black/white image.
|
||||
// Inspect the exact response displayed by the UI, without requesting a
|
||||
// newer latest.jpg or bypassing its normal authentication/CORS behavior.
|
||||
const response = screencastResponses.get(state.screencastImageUrl);
|
||||
if (!response) throw new Error('Displayed screencast has no observed image response');
|
||||
const imageData = `data:image/jpeg;base64,${(await response.buffer()).toString('base64')}`;
|
||||
const hasContent = await page.evaluate(async (data) => {
|
||||
const img = new Image();
|
||||
img.src = data;
|
||||
await img.decode();
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = canvas.height = 64;
|
||||
const context = canvas.getContext('2d');
|
||||
// Ignore the outer edges: a site header alone can appear while the
|
||||
// actual page is still blank. Marketing captures need page content.
|
||||
context.drawImage(img, img.naturalWidth * 0.1, img.naturalHeight * 0.1,
|
||||
img.naturalWidth * 0.8, img.naturalHeight * 0.8, 0, 0, 64, 64);
|
||||
const pixels = context.getImageData(0, 0, 64, 64).data;
|
||||
// Check each channel separately: a solid colored frame is also blank.
|
||||
return [0, 1, 2].some((channel) => {
|
||||
let min = 255;
|
||||
let max = 0;
|
||||
for (let i = channel; i < pixels.length; i += 4) {
|
||||
min = Math.min(min, pixels[i]);
|
||||
max = Math.max(max, pixels[i]);
|
||||
}
|
||||
return max - min > 16;
|
||||
});
|
||||
}, imageData);
|
||||
if (hasContent) {
|
||||
const current = await page.evaluate(liveProgressState);
|
||||
if (current.ready && current.screencastImageUrl === state.screencastImageUrl) {
|
||||
return {...current, screencastImageHasContent: true};
|
||||
}
|
||||
} else {
|
||||
rejectedFrameUrl = state.screencastImageUrl;
|
||||
}
|
||||
}
|
||||
throw new Error('Live progress deadline expired');
|
||||
} catch (error) {
|
||||
const state = await page.evaluate(liveProgressState).catch(() => ({url: page.url()}));
|
||||
if (rejectedFrameUrl) state.rejectedBlankFrameUrl = rejectedFrameUrl;
|
||||
throw new Error(`Live progress with a decoded screencast frame was not visible within 60 seconds: ${JSON.stringify(state)}`, {cause: error});
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.argv.includes('--help') || process.argv.includes('-h')) {
|
||||
usage();
|
||||
@@ -121,6 +228,7 @@ async function main() {
|
||||
const width = Number(process.env.SCREENSHOT_WIDTH || 1600);
|
||||
const height = Number(process.env.SCREENSHOT_HEIGHT || 1400);
|
||||
const fullPage = process.env.SCREENSHOT_FULL_PAGE === '1';
|
||||
const requireLiveProgress = process.env.SCREENSHOT_REQUIRE_LIVE_PROGRESS === '1';
|
||||
const variants = process.env.SCREENSHOT_VARIANTS_JSON
|
||||
? JSON.parse(process.env.SCREENSHOT_VARIANTS_JSON)
|
||||
: [];
|
||||
@@ -162,6 +270,14 @@ async function main() {
|
||||
await page.setCacheEnabled(false);
|
||||
await Promise.all(restoredPages.map((restoredPage) => restoredPage.close()));
|
||||
page.setDefaultTimeout(45000);
|
||||
const screencastResponses = new Map();
|
||||
if (requireLiveProgress) {
|
||||
page.on('response', (response) => {
|
||||
if (/\/files\/chrome_screencast\/latest\.jpg$/.test(new URL(response.url()).pathname) && response.ok()) {
|
||||
screencastResponses.set(response.url(), response);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const client = await page.createCDPSession();
|
||||
const documentRequests = new Map();
|
||||
@@ -209,6 +325,7 @@ async function main() {
|
||||
await page.setCookie(cookie);
|
||||
}
|
||||
|
||||
const liveProgressDeadline = Date.now() + 60000;
|
||||
let navigationResponse = await page.goto(url, { waitUntil: 'load', timeout: 60000 });
|
||||
|
||||
if (process.env.SCREENSHOT_LOGIN_USERNAME || process.env.SCREENSHOT_LOGIN_PASSWORD) {
|
||||
@@ -231,10 +348,22 @@ async function main() {
|
||||
}
|
||||
|
||||
await page.waitForSelector('body');
|
||||
await page.waitForSelector('#progress-monitor, #add-form', { timeout: 5000 }).catch(() => {});
|
||||
if (process.env.SCREENSHOT_WAIT_FOR_TEXT) {
|
||||
await page.waitForFunction((text) => document.body.innerText.includes(text), {}, process.env.SCREENSHOT_WAIT_FOR_TEXT);
|
||||
}
|
||||
if (process.env.SCREENSHOT_WAIT_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_WAIT_SELECTOR, { timeout: 45000 });
|
||||
}
|
||||
if (process.env.SCREENSHOT_PREVIEW_WAIT_FOR_TEXT) {
|
||||
const previewHandle = await page.waitForSelector('#main-frame', {timeout: 45000});
|
||||
const preview = await previewHandle.contentFrame();
|
||||
if (!preview) throw new Error('Snapshot preview iframe has no document');
|
||||
await preview.waitForFunction(
|
||||
(text) => document.body?.innerText.includes(text),
|
||||
{timeout: 45000},
|
||||
process.env.SCREENSHOT_PREVIEW_WAIT_FOR_TEXT,
|
||||
);
|
||||
}
|
||||
if (process.env.SCREENSHOT_CLICK_SELECTOR) {
|
||||
await page.waitForSelector(process.env.SCREENSHOT_CLICK_SELECTOR, { timeout: 45000 });
|
||||
await page.click(process.env.SCREENSHOT_CLICK_SELECTOR);
|
||||
@@ -249,7 +378,18 @@ async function main() {
|
||||
}, process.env.SCREENSHOT_SCROLL_SELECTOR);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1200));
|
||||
if (requireLiveProgress) {
|
||||
await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
|
||||
} else {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1200));
|
||||
}
|
||||
|
||||
// Selecting a hash opens its output stack after the saved header preference
|
||||
// is read. Apply the requested collapse through the real toggle afterwards.
|
||||
if (process.env.SCREENSHOT_SNAPSHOT_HEADER === 'collapsed') {
|
||||
const toggle = await page.$('.header-toggle[aria-expanded="true"]');
|
||||
if (toggle) await toggle.click();
|
||||
}
|
||||
|
||||
const checks = await page.evaluate(() => ({
|
||||
url: location.href,
|
||||
@@ -301,20 +441,26 @@ async function main() {
|
||||
}
|
||||
|
||||
const screenshotPaths = [];
|
||||
if (requireLiveProgress) checks.liveProgressCaptures = [];
|
||||
const capture = async (screenshotPath) => {
|
||||
await waitForVisibleCardPreviews(page, requireLiveProgress ? Math.max(1, liveProgressDeadline - Date.now()) : 45000);
|
||||
if (requireLiveProgress) {
|
||||
const liveProgress = await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
|
||||
checks.liveProgressCaptures.push({path: screenshotPath, viewport: page.viewport(), ...liveProgress});
|
||||
}
|
||||
await page.screenshot({path: screenshotPath, fullPage});
|
||||
screenshotPaths.push(screenshotPath);
|
||||
};
|
||||
if (variants.length) {
|
||||
for (const variant of variants) {
|
||||
const variantPath = path.resolve(variant.path);
|
||||
fs.mkdirSync(path.dirname(variantPath), { recursive: true });
|
||||
await page.setViewport({width: variant.width, height: variant.height, deviceScaleFactor: 1});
|
||||
await new Promise((resolve) => setTimeout(resolve, 300));
|
||||
await waitForVisibleCardPreviews(page);
|
||||
await page.screenshot({ path: variantPath, fullPage });
|
||||
screenshotPaths.push(variantPath);
|
||||
await capture(variantPath);
|
||||
}
|
||||
} else {
|
||||
await waitForVisibleCardPreviews(page);
|
||||
await page.screenshot({ path: output, fullPage });
|
||||
screenshotPaths.push(output);
|
||||
await capture(output);
|
||||
}
|
||||
console.log(JSON.stringify({ screenshotPath: screenshotPaths[0], screenshotPaths, checks }, null, 2));
|
||||
} finally {
|
||||
|
||||
@@ -54,7 +54,6 @@ version = 2
|
||||
"16f996904f0eaa96-1" = "run"
|
||||
"167cab3db82a6aee-1" = "run"
|
||||
"6a6234b63cd3d0ae-1" = "run"
|
||||
"396490664ce459e9-1" = "run"
|
||||
|
||||
# archivebox/mcp/README.md
|
||||
"d9ff84abeb97d380-1" = "run"
|
||||
@@ -69,7 +68,6 @@ version = 2
|
||||
"ac2c1555d3b9cd8f-1" = "run"
|
||||
"11a520069efa3387-1" = "run"
|
||||
"46fe9d81940ea385-1" = "run"
|
||||
"396490664ce459e9-2" = "run"
|
||||
|
||||
# docs/ArchiveBox-Architecture-Diagrams.md
|
||||
"7442dd337fd0a1dd-1" = "illustration"
|
||||
@@ -282,7 +280,6 @@ version = 2
|
||||
"16f996904f0eaa96-1" = "project"
|
||||
"167cab3db82a6aee-1" = "system"
|
||||
"6a6234b63cd3d0ae-1" = "collection"
|
||||
"396490664ce459e9-1" = "project"
|
||||
"d9ff84abeb97d380-1" = "collection"
|
||||
"aebc15190116f5cd-1" = "collection"
|
||||
"0b307d4f1033723f-1" = "collection"
|
||||
@@ -290,7 +287,6 @@ version = 2
|
||||
"ac2c1555d3b9cd8f-1" = "project"
|
||||
"11a520069efa3387-1" = "system"
|
||||
"46fe9d81940ea385-1" = "collection"
|
||||
"396490664ce459e9-2" = "project"
|
||||
"201988374bff6d14-1" = "collection"
|
||||
"e305c234db136566-1" = "collection"
|
||||
"798793d9d6b2e12e-1" = "collection"
|
||||
@@ -363,7 +359,6 @@ version = 2
|
||||
"16f996904f0eaa96-1" = "ubuntu"
|
||||
"167cab3db82a6aee-1" = "ubuntu"
|
||||
"6a6234b63cd3d0ae-1" = "ubuntu"
|
||||
"396490664ce459e9-1" = "ubuntu"
|
||||
"d9ff84abeb97d380-1" = "ubuntu"
|
||||
"aebc15190116f5cd-1" = "ubuntu"
|
||||
"0b307d4f1033723f-1" = "ubuntu"
|
||||
@@ -371,7 +366,6 @@ version = 2
|
||||
"ac2c1555d3b9cd8f-1" = "ubuntu"
|
||||
"11a520069efa3387-1" = "ubuntu"
|
||||
"46fe9d81940ea385-1" = "ubuntu"
|
||||
"396490664ce459e9-2" = "ubuntu"
|
||||
"201988374bff6d14-1" = "ubuntu"
|
||||
"e305c234db136566-1" = "ubuntu"
|
||||
"798793d9d6b2e12e-1" = "ubuntu"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "archivebox",
|
||||
"version": "0.9.71rc3",
|
||||
"version": "0.9.71rc42",
|
||||
"repository": "github:ArchiveBox/ArchiveBox",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
|
||||
@@ -158,7 +158,7 @@
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
|
||||
<h3>يحفظ علامات التبويب في ثوان ، يخزنها منذ عقود</h3>
|
||||
<p>جمع الإشارات المرجعية للقراءة لاحقًا ، أو الاحتفاظ بالأدلة للحالات القانونية التي يتم التحقق منها من عمليات التقاط من موقعنا <a href="https://tlsnotary.zervice.io/">ملحق تسجيل TLSNotary</a>.</p>
|
||||
<p>جمع الإشارات المرجعية للقراءة لاحقًا ، أو الاحتفاظ بالأدلة للحالات القانونية التي يتم التحقق منها من عمليات التقاط من موقعنا <a href="https://tlsnotary.archivebox.io/">ملحق تسجيل TLSNotary</a>.</p>
|
||||
</article>
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
|
||||
@@ -204,7 +204,7 @@
|
||||
<ul>
|
||||
<li>تخزين لقطات الشاشة ، ملفات PDFs ، الرؤوس ، ملفات WARC ، واستخراج النص.</li>
|
||||
<li>ضع علامة على المصادر واستعراضها من خلال واجهة الويب المستضافة ذاتيًا.</li>
|
||||
<li>استخدام ZK proofs مع <a href="https://tlsnotary.zervice.io/">TLSNotary</a> لإثبات صحة المحتوى.</li>
|
||||
<li>استخدام ZK proofs مع <a href="https://tlsnotary.archivebox.io/">TLSNotary</a> لإثبات صحة المحتوى.</li>
|
||||
</ul>
|
||||
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="مخرج TLSNotary للتحقق من استجابة تم أرشيفة في صفحة تفاصيل اللقطة" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
|
||||
</article>
|
||||
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
|
||||
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>خادم منزلي</span></div></div><figcaption>على الأجهزة الخاصة بك</figcaption></figure>
|
||||
<div class="command-card">
|
||||
<div class="step"><span>1</span><div><h4>اختر منصتك</h4><p>قد تتأخر تكاملات المجتمع عن إصدارات Docker وPython.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">نونريد</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>تثبيت وفتح ArchiveBox</h4><p>اتبع دليل تثبيت المنصة ، قم بتكوين التخزين المستمر ، ثم افتح التطبيق لإنهاء الإعداد.</p><p>بالنسبة إلى TrueNAS، راجع <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">ملاحظات منصة README</a> قبل اختيار التكامل.</p></div></div>
|
||||
<div class="step"><span>1</span><div><h4>اختر منصتك</h4><p>قد تتأخر تكاملات المجتمع عن إصدارات Docker وPython.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">نونريد</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>تثبيت وفتح ArchiveBox</h4><p>اتبع دليل تثبيت المنصة ، قم بتكوين التخزين المستمر ، ثم افتح التطبيق لإنهاء الإعداد.</p></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
|
||||
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<li><a aria-label="ملحق أحادي الملف" href="https://plugins.archivebox.io/#singlefile" title="ملف فردي"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
|
||||
<li><a aria-label="يوتيوب/yt-dlp البرنامج المساعد" href="https://plugins.archivebox.io/#ytdlp" title="يوتيوب/yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
|
||||
<li><a aria-label="ذيل" href="https://tailscale.com/" title="ذيل"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSNotary" href="https://tlsnotary.zervice.io/" title="TLSNotary"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSNotary" href="https://tlsnotary.archivebox.io/" title="TLSNotary"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
</ul>
|
||||
<p class="configuration-description">يمكن إعداد ArchiveBox عبر واجهة الإدارة أو واجهة سطر الأوامر، <code>ArchiveBox.conf</code>، أو عن طريق استيراد الإعدادات وملفات الكوكيز من Chrome أو متصفحات أخرى. يعمل نموذج التكوين نفسه في Docker وDocker Compose، وفي عمليات التثبيت على الأجهزة الحقيقية، وفي المهام المجدولة، وكذلك في التشغيلات المؤقتة عبر واجهة سطر الأوامر.</p>
|
||||
</div>
|
||||
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<!-- ARCHIVEBOX:FOOTER-START -->
|
||||
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">أدوات مفتوحة المصدر للاحتفاظ بنسخة دائمة من الويب.<br/>بياناتك. أجهزتك. أرشيفك.</p></div><p class="abx-footer-motto">مكان للويب<br/>الذي تريد الاحتفاظ به.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>التطبيقات والأدوات</h2><ul><li><a href="https://archivebox.io/">خادم ArchiveBox</a><a aria-label="خادم ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox لنظامَي macOS وiOS</a><a aria-label="ArchiveBox لنظامَي macOS وiOS — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox لنظام Android</a><a aria-label="ArchiveBox لنظام Android — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox لسطح المكتب</a><a aria-label="ArchiveBox لسطح المكتب — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">إضافة ArchiveBox للمتصفح</a><a aria-label="إضافة ArchiveBox للمتصفح — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">مكتبة إضافات ArchiveBox</a><a aria-label="مكتبة إضافات ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-Line truncated
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>التطبيقات والأدوات</h2><ul><li><a href="https://archivebox.io/">خادم ArchiveBox</a><a aria-label="خادم ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox لنظامَي macOS وiOS</a><a aria-label="ArchiveBox لنظامَي macOS وiOS — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox لنظام Android</a><a aria-label="ArchiveBox لنظام Android — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox لسطح المكتب</a><a aria-label="ArchiveBox لسطح المكتب — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">إضافة ArchiveBox للمتصفح</a><a aria-label="إضافة ArchiveBox للمتصفح — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">مكتبة إضافات ArchiveBox</a><a aria-label="مكتبة إضافات ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-Line truncated
|
||||
<!-- /ARCHIVEBOX:FOOTER-START -->
|
||||
<div class="abx-footer-local"><p>أرشفة ويب مفتوحة المصدر ومستضافة ذاتيًا. بترخيص MIT وبدعم غير ربحي ومصممة لمجموعات تدوم طويلًا.</p>
|
||||
<div>
|
||||
|
||||
@@ -158,7 +158,7 @@
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
|
||||
<h3>Guarda pestañas en segundos, las almacena durante décadas</h3>
|
||||
<p>Recopile marcadores para leer más tarde o conserve pruebas para casos legales con capturas verificables de nuestra <a href="https://tlsnotary.zervice.io/">Complemento de firma TLSNotary</a>.</p>
|
||||
<p>Recopile marcadores para leer más tarde o conserve pruebas para casos legales con capturas verificables de nuestra <a href="https://tlsnotary.archivebox.io/">Complemento de firma TLSNotary</a>.</p>
|
||||
</article>
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
|
||||
@@ -204,7 +204,7 @@
|
||||
<ul>
|
||||
<li>Almacene capturas de pantalla, archivos PDF, encabezados, archivos WARC y extracción de texto.</li>
|
||||
<li>Etiqueta y revisa fuentes a través de la interfaz web autohospedada.</li>
|
||||
<li>Utilice pruebas ZK con <a href="https://tlsnotary.zervice.io/">TLSNotario</a> para demostrar la autenticidad del contenido.</li>
|
||||
<li>Utilice pruebas ZK con <a href="https://tlsnotary.archivebox.io/">TLSNotario</a> para demostrar la autenticidad del contenido.</li>
|
||||
</ul>
|
||||
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="Salida de TLSNotary que verifica una respuesta archivada en la página de detalles de la instantánea" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
|
||||
</article>
|
||||
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
|
||||
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Servidor doméstico</span></div></div><figcaption>En tu propio hardware</figcaption></figure>
|
||||
<div class="command-card">
|
||||
<div class="step"><span>1</span><div><h4>Elige tu plataforma</h4><p>Las integraciones comunitarias pueden ir por detrás de las versiones de Docker y Python.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">sin asaltar</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Instalar y abrir ArchiveBox</h4><p>Siga la guía de instalación de la plataforma, configure el almacenamiento persistente y luego abra la aplicación para finalizar la configuración.</p><p>Para TrueNAS, consulte la <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">Notas de la plataforma README</a> antes de elegir una integración.</p></div></div>
|
||||
<div class="step"><span>1</span><div><h4>Elige tu plataforma</h4><p>Las integraciones comunitarias pueden ir por detrás de las versiones de Docker y Python.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">sin asaltar</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Instalar y abrir ArchiveBox</h4><p>Siga la guía de instalación de la plataforma, configure el almacenamiento persistente y luego abra la aplicación para finalizar la configuración.</p></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
|
||||
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<li><a aria-label="Complemento de archivo único" href="https://plugins.archivebox.io/#singlefile" title="Archivo único"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
|
||||
<li><a aria-label="Complemento YouTube/yt-dlp" href="https://plugins.archivebox.io/#ytdlp" title="YouTube/yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
|
||||
<li><a aria-label="Escala de cola" href="https://tailscale.com/" title="Escala de cola"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSNotario" href="https://tlsnotary.zervice.io/" title="TLSNotario"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSNotario" href="https://tlsnotary.archivebox.io/" title="TLSNotario"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
</ul>
|
||||
<p class="configuration-description">ArchiveBox se puede configurar a través de la interfaz de usuario de administración, CLI, <code>ArchiveBox.conf</code>, o importando configuraciones y cookies desde Chrome u otros navegadores. El mismo modelo de configuración funciona en Docker, Docker Compose, instalaciones básicas, trabajos programados y ejecuciones CLI únicas.</p>
|
||||
</div>
|
||||
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<!-- ARCHIVEBOX:FOOTER-START -->
|
||||
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">Herramientas de código abierto para conservar una copia duradera de la web.<br/>Tus datos. Tus dispositivos. Tu archivo.</p></div><p class="abx-footer-motto">Un hogar para la web<br/>que quieres conservar.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Aplicaciones y herramientas</h2><ul><li><a href="https://archivebox.io/">Servidor ArchiveBox</a><a aria-label="Servidor ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox para macOS e iOS</a><a aria-label="ArchiveBox para macOS e iOS — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox para Android</a><a aria-label="ArchiveBox para Android — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox para escritorio</a><a aria-label="ArchiveBox para escritorio — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extensión de navegador ArchiveBox</a><a aria-label="Extensión de navegador ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Biblioteca de complementos ArchiveBox</a><a aria-label="Biblioteca de complementos ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2Line truncated
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Aplicaciones y herramientas</h2><ul><li><a href="https://archivebox.io/">Servidor ArchiveBox</a><a aria-label="Servidor ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox para macOS e iOS</a><a aria-label="ArchiveBox para macOS e iOS — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox para Android</a><a aria-label="ArchiveBox para Android — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox para escritorio</a><a aria-label="ArchiveBox para escritorio — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extensión de navegador ArchiveBox</a><a aria-label="Extensión de navegador ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Biblioteca de complementos ArchiveBox</a><a aria-label="Biblioteca de complementos ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2Line truncated
|
||||
<!-- /ARCHIVEBOX:FOOTER-START -->
|
||||
<div class="abx-footer-local"><p>Archivado web de código abierto y autoalojado. Con licencia MIT, apoyo de organizaciones sin ánimo de lucro y diseñado para colecciones duraderas.</p>
|
||||
<div>
|
||||
|
||||
@@ -158,7 +158,7 @@
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
|
||||
<h3>Enregistre les onglets en quelques secondes, les stocke pendant des décennies</h3>
|
||||
<p>Collectez des signets pour les lire plus tard ou conservez des preuves pour des affaires juridiques avec des captures vérifiables de notre <a href="https://tlsnotary.zervice.io/">Plugin de signature TLSNotary</a>.</p>
|
||||
<p>Collectez des signets pour les lire plus tard ou conservez des preuves pour des affaires juridiques avec des captures vérifiables de notre <a href="https://tlsnotary.archivebox.io/">Plugin de signature TLSNotary</a>.</p>
|
||||
</article>
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
|
||||
@@ -204,7 +204,7 @@
|
||||
<ul>
|
||||
<li>Stockez des captures d'écran, des PDF, des en-têtes, des fichiers WARC et des extractions de texte.</li>
|
||||
<li>Marquez et examinez les sources via l’interface Web auto-hébergée.</li>
|
||||
<li>Utilisez des preuves ZK avec <a href="https://tlsnotary.zervice.io/">TLSNotaire</a> pour prouver l'authenticité du contenu.</li>
|
||||
<li>Utilisez des preuves ZK avec <a href="https://tlsnotary.archivebox.io/">TLSNotaire</a> pour prouver l'authenticité du contenu.</li>
|
||||
</ul>
|
||||
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="Sortie TLSNotary vérifiant une réponse archivée sur la page de détails de l'instantané" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
|
||||
</article>
|
||||
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
|
||||
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Serveur domestique</span></div></div><figcaption>Sur votre propre matériel</figcaption></figure>
|
||||
<div class="command-card">
|
||||
<div class="step"><span>1</span><div><h4>Choisissez votre plateforme</h4><p>Les intégrations communautaires peuvent être en retard par rapport aux versions Docker et Python.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">Sans raid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Installer et ouvrir ArchiveBox</h4><p>Suivez le guide d'installation de la plateforme, configurez le stockage persistant, puis ouvrez l'application pour terminer la configuration.</p><p>Pour TrueNAS, consultez le <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">Notes de la plateforme README</a> avant de choisir une intégration.</p></div></div>
|
||||
<div class="step"><span>1</span><div><h4>Choisissez votre plateforme</h4><p>Les intégrations communautaires peuvent être en retard par rapport aux versions Docker et Python.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">Sans raid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Installer et ouvrir ArchiveBox</h4><p>Suivez le guide d'installation de la plateforme, configurez le stockage persistant, puis ouvrez l'application pour terminer la configuration.</p></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
|
||||
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<li><a aria-label="Plugin SingleFile" href="https://plugins.archivebox.io/#singlefile" title="Fichier unique"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
|
||||
<li><a aria-label="Plugin YouTube/yt-dlp" href="https://plugins.archivebox.io/#ytdlp" title="YouTube / yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
|
||||
<li><a aria-label="Échelle arrière" href="https://tailscale.com/" title="Échelle arrière"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSNotaire" href="https://tlsnotary.zervice.io/" title="TLSNotaire"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSNotaire" href="https://tlsnotary.archivebox.io/" title="TLSNotaire"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
</ul>
|
||||
<p class="configuration-description">ArchiveBox peut être configuré via l'interface utilisateur d'administration, la CLI, <code>ArchiveBox.conf</code>, ou en important la configuration et les cookies depuis Chrome/d'autres navigateurs. Le même modèle de configuration fonctionne dans Docker, Docker Compose, les installations nues, les tâches planifiées et les exécutions CLI ponctuelles.</p>
|
||||
</div>
|
||||
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<!-- ARCHIVEBOX:FOOTER-START -->
|
||||
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">Des outils open source pour conserver durablement une copie du Web.<br/>Vos données. Vos appareils. Vos archives.</p></div><p class="abx-footer-motto">Un espace pour le Web<br/>que vous souhaitez conserver.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Applications et outils</h2><ul><li><a href="https://archivebox.io/">Serveur ArchiveBox</a><a aria-label="Serveur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox pour macOS et iOS</a><a aria-label="ArchiveBox pour macOS et iOS — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox pour Android</a><a aria-label="ArchiveBox pour Android — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox pour ordinateur</a><a aria-label="ArchiveBox pour ordinateur — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extension de navigateur ArchiveBox</a><a aria-label="Extension de navigateur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Bibliothèque de plugins ArchiveBox</a><a aria-label="Bibliothèque de plugins ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.9Line truncated
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Applications et outils</h2><ul><li><a href="https://archivebox.io/">Serveur ArchiveBox</a><a aria-label="Serveur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox pour macOS et iOS</a><a aria-label="ArchiveBox pour macOS et iOS — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox pour Android</a><a aria-label="ArchiveBox pour Android — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox pour ordinateur</a><a aria-label="ArchiveBox pour ordinateur — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extension de navigateur ArchiveBox</a><a aria-label="Extension de navigateur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Bibliothèque de plugins ArchiveBox</a><a aria-label="Bibliothèque de plugins ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.9Line truncated
|
||||
<!-- /ARCHIVEBOX:FOOTER-START -->
|
||||
<div class="abx-footer-local"><p>Archivage web open source et auto-hébergé. Sous licence MIT, soutenu par des organismes à but non lucratif et conçu pour des collections durables.</p>
|
||||
<div>
|
||||
|
||||
@@ -149,7 +149,7 @@
|
||||
<article>
|
||||
<span class="card-icon" aria-hidden="true"><svg viewBox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"/></svg></span>
|
||||
<h3>Saves tabs in seconds, stores them for decades</h3>
|
||||
<p>Collect bookmarks to read later, or preserve evidence for legal cases with verifiable captures from our <a href="https://tlsnotary.zervice.io/">TLSNotary signing plugin</a>.</p>
|
||||
<p>Collect bookmarks to read later, or preserve evidence for legal cases with verifiable captures from our <a href="https://tlsnotary.archivebox.io/">TLSNotary signing plugin</a>.</p>
|
||||
</article>
|
||||
<article>
|
||||
<span class="card-icon" aria-hidden="true"><svg viewBox="0 0 24 24"><circle cx="10" cy="10" r="7"/><path d="m15 15 6 6"/></svg></span>
|
||||
@@ -196,7 +196,7 @@
|
||||
<ul>
|
||||
<li>Store screenshots, PDFs, headers, WARC files, and text extraction.</li>
|
||||
<li>Tag and review sources through the self-hosted web interface.</li>
|
||||
<li>Use ZK proofs with <a href="https://tlsnotary.zervice.io/">TLSNotary</a> to prove content authenticity.</li>
|
||||
<li>Use ZK proofs with <a href="https://tlsnotary.archivebox.io/">TLSNotary</a> to prove content authenticity.</li>
|
||||
</ul>
|
||||
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600" height="1000" loading="lazy" alt="TLSNotary output verifying an archived response on the snapshot details page"></a></figure>
|
||||
</article>
|
||||
@@ -363,8 +363,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<div class="install-panel" id="install-platforms" aria-labelledby="install-tab-platforms">
|
||||
<figure class="install-art install-art--server"><div class="install-art-stage"><svg class="install-server-rack" viewBox="0 0 160 180" aria-hidden="true"><rect x="18" y="12" width="124" height="156" rx="12"/><rect x="32" y="28" width="96" height="34" rx="5"/><rect x="32" y="73" width="96" height="34" rx="5"/><rect x="32" y="118" width="96" height="34" rx="5"/><path d="M47 45h43m-43 45h43m-43 45h43"/><circle cx="111" cy="45" r="4"/><circle cx="111" cy="90" r="4"/><circle cx="111" cy="135" r="4"/></svg><div class="install-art-tile"><img src="./assets/install/archivebox-server.png" alt="" class="" loading="lazy"><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Home server</span></div></div><figcaption>On your own hardware</figcaption></figure>
|
||||
<div class="command-card">
|
||||
<div class="step"><span>1</span><div><h4>Choose your platform</h4><p>Community integrations may lag behind the Docker and Python releases.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">Unraid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Install and open ArchiveBox</h4><p>Follow the platform’s installation guide, configure persistent storage, then open the app to finish setup.</p><p>For TrueNAS, see the <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">README’s platform notes</a> before choosing an integration.</p></div></div>
|
||||
<div class="step"><span>1</span><div><h4>Choose your platform</h4><p>Community integrations may lag behind the Docker and Python releases.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">Unraid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Install and open ArchiveBox</h4><p>Follow the platform’s installation guide, configure persistent storage, then open the app to finish setup.</p></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="install-panel" id="install-hosting" aria-labelledby="install-tab-hosting">
|
||||
@@ -402,7 +402,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<li><a href="https://plugins.archivebox.io/#singlefile" aria-label="SingleFile plugin" title="SingleFile"><img src="./assets/plugins/singlefile.png" width="44" height="44" loading="lazy" alt=""></a></li>
|
||||
<li><a href="https://plugins.archivebox.io/#ytdlp" aria-label="YouTube / yt-dlp plugin" title="YouTube / yt-dlp"><img src="./assets/plugins/youtube.svg" width="44" height="44" loading="lazy" alt=""></a></li>
|
||||
<li><a href="https://tailscale.com/" aria-label="Tailscale" title="Tailscale"><img src="./assets/plugins/tailscale.svg" width="44" height="44" loading="lazy" alt=""></a></li>
|
||||
<li><a href="https://tlsnotary.zervice.io/" aria-label="TLSNotary" title="TLSNotary"><img src="./assets/plugins/tlsnotary.svg" width="44" height="44" loading="lazy" alt=""></a></li>
|
||||
<li><a href="https://tlsnotary.archivebox.io/" aria-label="TLSNotary" title="TLSNotary"><img src="./assets/plugins/tlsnotary.svg" width="44" height="44" loading="lazy" alt=""></a></li>
|
||||
</ul>
|
||||
<p class="configuration-description">ArchiveBox can be configured via the admin UI, CLI, <code>ArchiveBox.conf</code>, or by importing config and cookies from Chrome / other browsers. The same configuration model works in Docker, Docker Compose, bare-metal installs, scheduled jobs, and one-off CLI runs.</p>
|
||||
</div>
|
||||
|
||||
@@ -158,7 +158,7 @@
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
|
||||
<h3>Сохраняет вкладки за считанные секунды и хранит их десятилетиями.</h3>
|
||||
<p>Собирайте закладки, чтобы прочитать их позже, или сохраняйте доказательства по юридическим делам с помощью поддающихся проверке записей из наших <a href="https://tlsnotary.zervice.io/">Плагин подписи TLSNotary</a>.</p>
|
||||
<p>Собирайте закладки, чтобы прочитать их позже, или сохраняйте доказательства по юридическим делам с помощью поддающихся проверке записей из наших <a href="https://tlsnotary.archivebox.io/">Плагин подписи TLSNotary</a>.</p>
|
||||
</article>
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
|
||||
@@ -204,7 +204,7 @@
|
||||
<ul>
|
||||
<li>Храните снимки экрана, PDF-файлы, заголовки, файлы WARC и извлечение текста.</li>
|
||||
<li>Отмечайте и просматривайте источники через автономный веб-интерфейс.</li>
|
||||
<li>Используйте доказательства ZK с <a href="https://tlsnotary.zervice.io/">TLSНотариус</a> для подтверждения подлинности контента.</li>
|
||||
<li>Используйте доказательства ZK с <a href="https://tlsnotary.archivebox.io/">TLSНотариус</a> для подтверждения подлинности контента.</li>
|
||||
</ul>
|
||||
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="Вывод TLSNotary, подтверждающий заархивированный ответ на странице сведений о снимке" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
|
||||
</article>
|
||||
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
|
||||
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Домашний сервер</span></div></div><figcaption>На вашем собственном оборудовании</figcaption></figure>
|
||||
<div class="command-card">
|
||||
<div class="step"><span>1</span><div><h4>Выберите свою платформу</h4><p>Интеграция сообщества может отставать от выпусков Docker и Python.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">Не набег</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Установите и откройте ArchiveBox.</h4><p>Следуйте инструкциям по установке платформы, настройте постоянное хранилище, затем откройте приложение, чтобы завершить настройку.</p><p>Информацию о TrueNAS см. <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">Примечания к платформе README</a> прежде чем выбрать интеграцию.</p></div></div>
|
||||
<div class="step"><span>1</span><div><h4>Выберите свою платформу</h4><p>Интеграция сообщества может отставать от выпусков Docker и Python.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">Не набег</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>Установите и откройте ArchiveBox.</h4><p>Следуйте инструкциям по установке платформы, настройте постоянное хранилище, затем откройте приложение, чтобы завершить настройку.</p></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
|
||||
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<li><a aria-label="Плагин SingleFile" href="https://plugins.archivebox.io/#singlefile" title="Один файл"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
|
||||
<li><a aria-label="Плагин YouTube/yt-dlp" href="https://plugins.archivebox.io/#ytdlp" title="YouTube / yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
|
||||
<li><a aria-label="Хвостовая чешуя" href="https://tailscale.com/" title="Хвостовая чешуя"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSНотариус" href="https://tlsnotary.zervice.io/" title="TLSНотариус"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLSНотариус" href="https://tlsnotary.archivebox.io/" title="TLSНотариус"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
</ul>
|
||||
<p class="configuration-description">ArchiveBox можно настроить через пользовательский интерфейс администратора, CLI, <code>ArchiveBox.conf</code>или импортировав конфигурацию и файлы cookie из Chrome/других браузеров. Одна и та же модель конфигурации работает в Docker, Docker Compose, при установке на «голое железо», запланированных заданиях и одноразовых запусках CLI.</p>
|
||||
</div>
|
||||
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<!-- ARCHIVEBOX:FOOTER-START -->
|
||||
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">Открытые инструменты для долговременного сохранения копий веб-страниц.<br/>Ваши данные. Ваши устройства. Ваш архив.</p></div><p class="abx-footer-motto">Место для веб-страниц<br/>которые вы хотите сохранить.</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Приложения и инструменты</h2><ul><li><a href="https://archivebox.io/">Сервер ArchiveBox</a><a aria-label="Сервер ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox для macOS и iOS</a><a aria-label="ArchiveBox для macOS и iOS — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox для Android</a><a aria-label="ArchiveBox для Android — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox для компьютера</a><a aria-label="ArchiveBox для компьютера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Расширение ArchiveBox для браузера</a><a aria-label="Расширение ArchiveBox для браузера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Библиотека плагинов ArchiveBox</a><a aria-label="Библиотека плагинов ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.Line truncated
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Приложения и инструменты</h2><ul><li><a href="https://archivebox.io/">Сервер ArchiveBox</a><a aria-label="Сервер ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox для macOS и iOS</a><a aria-label="ArchiveBox для macOS и iOS — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox для Android</a><a aria-label="ArchiveBox для Android — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox для компьютера</a><a aria-label="ArchiveBox для компьютера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Расширение ArchiveBox для браузера</a><a aria-label="Расширение ArchiveBox для браузера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Библиотека плагинов ArchiveBox</a><a aria-label="Библиотека плагинов ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.Line truncated
|
||||
<!-- /ARCHIVEBOX:FOOTER-START -->
|
||||
<div class="abx-footer-local"><p>Открытое веб-архивирование на собственном сервере. Лицензия MIT, поддержка некоммерческих организаций и ориентация на долговременные коллекции.</p>
|
||||
<div>
|
||||
|
||||
@@ -158,7 +158,7 @@
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
|
||||
<h3>在几秒钟内保存选项卡,将其存储数十年</h3>
|
||||
<p>收集书签以供稍后阅读,或通过我们的可验证捕获保存法律案件的证据 <a href="https://tlsnotary.zervice.io/">TLSNotary 签名插件</a>.</p>
|
||||
<p>收集书签以供稍后阅读,或通过我们的可验证捕获保存法律案件的证据 <a href="https://tlsnotary.archivebox.io/">TLSNotary 签名插件</a>.</p>
|
||||
</article>
|
||||
<article>
|
||||
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
|
||||
@@ -204,7 +204,7 @@
|
||||
<ul>
|
||||
<li>存储屏幕截图、PDF、标题、WARC 文件和文本提取。</li>
|
||||
<li>通过自托管 Web 界面标记和审查源。</li>
|
||||
<li>使用 ZK 证明 <a href="https://tlsnotary.zervice.io/">TLS公证人</a> 以证明内容的真实性。</li>
|
||||
<li>使用 ZK 证明 <a href="https://tlsnotary.archivebox.io/">TLS公证人</a> 以证明内容的真实性。</li>
|
||||
</ul>
|
||||
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="TLSNotary 输出验证快照详细信息页面上的存档响应" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
|
||||
</article>
|
||||
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
|
||||
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>家庭服务器</span></div></div><figcaption>在您自己的硬件上</figcaption></figure>
|
||||
<div class="command-card">
|
||||
<div class="step"><span>1</span><div><h4>选择您的平台</h4><p>社区集成可能落后于 Docker 和 Python 版本。</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">解除突袭</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>安装并打开 ArchiveBox</h4><p>按照平台的安装指南,配置持久存储,然后打开应用程序完成设置。</p><p>对于 TrueNAS,请参阅 <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">README 的平台说明</a> 在选择集成之前。</p></div></div>
|
||||
<div class="step"><span>1</span><div><h4>选择您的平台</h4><p>社区集成可能落后于 Docker 和 Python 版本。</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">解除突袭</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
|
||||
<div class="step"><span>2</span><div><h4>安装并打开 ArchiveBox</h4><p>按照平台的安装指南,配置持久存储,然后打开应用程序完成设置。</p></div></div>
|
||||
</div>
|
||||
</div>
|
||||
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
|
||||
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<li><a aria-label="单文件插件" href="https://plugins.archivebox.io/#singlefile" title="单一文件"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
|
||||
<li><a aria-label="YouTube / yt-dlp 插件" href="https://plugins.archivebox.io/#ytdlp" title="YouTube / yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
|
||||
<li><a aria-label="尾鳞" href="https://tailscale.com/" title="尾鳞"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLS公证人" href="https://tlsnotary.zervice.io/" title="TLS公证人"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
<li><a aria-label="TLS公证人" href="https://tlsnotary.archivebox.io/" title="TLS公证人"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
|
||||
</ul>
|
||||
<p class="configuration-description">ArchiveBox 可以通过管理 UI、CLI 进行配置, <code>ArchiveBox.conf</code>,或通过从 Chrome/其他浏览器导入配置和 cookie。相同的配置模型适用于 Docker、Docker Compose、裸机安装、计划作业和一次性 CLI 运行。</p>
|
||||
</div>
|
||||
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
|
||||
<!-- ARCHIVEBOX:FOOTER-START -->
|
||||
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
|
||||
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">用于长期保存网页副本的开源工具。<br/>你的数据。你的设备。你的归档。</p></div><p class="abx-footer-motto">为网页安个家<br/>保存你想保留的内容。</p></div>
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>应用与工具</h2><ul><li><a href="https://archivebox.io/">ArchiveBox 服务器</a><a aria-label="ArchiveBox 服务器 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">macOS 和 iOS 版 ArchiveBox</a><a aria-label="macOS 和 iOS 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">Android 版 ArchiveBox</a><a aria-label="Android 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox 桌面版</a><a aria-label="ArchiveBox 桌面版 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox 浏览器扩展</a><a aria-label="ArchiveBox 浏览器扩展 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox 插件库</a><a aria-label="ArchiveBox 插件库 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z">Line truncated
|
||||
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>应用与工具</h2><ul><li><a href="https://archivebox.io/">ArchiveBox 服务器</a><a aria-label="ArchiveBox 服务器 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">macOS 和 iOS 版 ArchiveBox</a><a aria-label="macOS 和 iOS 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">Android 版 ArchiveBox</a><a aria-label="Android 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox 桌面版</a><a aria-label="ArchiveBox 桌面版 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox 浏览器扩展</a><a aria-label="ArchiveBox 浏览器扩展 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox 插件库</a><a aria-label="ArchiveBox 插件库 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z">Line truncated
|
||||
<!-- /ARCHIVEBOX:FOOTER-START -->
|
||||
<div class="abx-footer-local"><p>开源、自托管的网页归档。采用 MIT 许可证,由非营利组织支持,为长期保存的收藏而设计。</p>
|
||||
<div>
|
||||
|
||||
+6
-6
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "archivebox"
|
||||
version = "0.9.71rc3"
|
||||
version = "0.9.71rc42"
|
||||
requires-python = ">=3.13"
|
||||
description = "Self-hosted internet archiving solution."
|
||||
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
|
||||
@@ -77,10 +77,10 @@ dependencies = [
|
||||
"w3lib>=2.2.1", # used for parsing content-type encoding from http response headers & html tags
|
||||
### Extractor dependencies (runtime binaries resolved through abxpkg)
|
||||
### Binary/Package Management
|
||||
"abxbus==2.5.65", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abxpkg==1.13.11", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abx-plugins[opencode]==1.13.82", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.13.59", # shared ArchiveBox downloader package
|
||||
"abxbus==2.5.68", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abxpkg==1.13.15", # direct imports only; version constrained by abx-dl -> abx-plugins
|
||||
"abx-plugins[opencode]==1.13.103", # optional AI runtime clients are owned by the plugin
|
||||
"abx-dl==1.13.71", # shared ArchiveBox downloader package
|
||||
### UUID7 backport for Python <3.14
|
||||
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
|
||||
"browser-cookie3>=0.20.1",
|
||||
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
|
||||
|
||||
|
||||
[tool.bumpver]
|
||||
current_version = "v0.9.71rc3"
|
||||
current_version = "v0.9.71rc42"
|
||||
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
|
||||
commit_message = "bump version {old_version} -> {new_version}"
|
||||
tag_message = "{new_version}"
|
||||
|
||||
@@ -68,9 +68,7 @@ archivebox run
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
|
||||
uv run prek run --all-files
|
||||
```
|
||||
Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
|
||||
with `uv run prek run --all-files`. For focused work, select the relevant tests.
|
||||
|
||||
Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
|
||||
@@ -13,18 +13,18 @@ supported-markers = [
|
||||
]
|
||||
|
||||
[options]
|
||||
exclude-newer = "2026-09-22T07:00:25.284838761Z"
|
||||
exclude-newer = "2026-09-23T08:33:14.714060607Z"
|
||||
exclude-newer-span = "P5D"
|
||||
|
||||
[options.exclude-newer-package]
|
||||
abxbus = { timestamp = "2026-09-27T07:00:24.284850353Z", span = "PT1S" }
|
||||
abxbus = { timestamp = "2026-09-28T08:33:13.714073511Z", span = "PT1S" }
|
||||
abx-plugins = "2100-01-01T00:00:00Z"
|
||||
abx-dl = "2100-01-01T00:00:00Z"
|
||||
abxpkg = "2100-01-01T00:00:00Z"
|
||||
|
||||
[[package]]
|
||||
name = "abx-dl"
|
||||
version = "1.13.59"
|
||||
version = "1.13.71"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -38,14 +38,14 @@ dependencies = [
|
||||
{ name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8c/e3/d4e9459cb2ad0ac0a37e4a0747bd3b0bab414294706e624a32fd0de19800/abx_dl-1.13.59.tar.gz", hash = "sha256:0c431e6e0dbc088ecc1625843405b89093be8e99c1e72672d9266e8cfbc41fb7", size = 97393 }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/79/67/9eaf667a429db72f0609b493aca107ff42f29a31dfd2333fcea02ece900a/abx_dl-1.13.71.tar.gz", hash = "sha256:1fdb6e584e2d4e7d7c04048cbd5c37bef3578352d6c2fe7ab76fc706ddadf09b", size = 97387 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/97/16/024bcbddc7c75fc3adbfdd7593374add9c95f46d76f55b2209185e2702f4/abx_dl-1.13.59-py3-none-any.whl", hash = "sha256:9bbce2ceebcbfe98473beb4d71d1aecfb1c8364a5fef3fef46f5cd2150f939d9", size = 102837 },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/23/d12ed879d5253ae6bbbf0e2ced77b0b59f3b4d4c3b381cc84518fee0443e/abx_dl-1.13.71-py3-none-any.whl", hash = "sha256:0585182839d8e9d3c93ee87d0ca6e2fa2d9d8ac936515f2700046324d210ce29", size = 102837 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "abx-plugins"
|
||||
version = "1.13.82"
|
||||
version = "1.13.103"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -54,9 +54,9 @@ dependencies = [
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ec/6f/c13daea79627fdd589aa2a932ecc9d82251229e1d836b76bc658d0f4b092/abx_plugins-1.13.82.tar.gz", hash = "sha256:64916aebd3db8aeb131f69ac6c6fa9b9b436944e174104a1e6652556cf4f43bb", size = 458681, upload-time = "2026-09-27T06:43:53.582Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5c/ba/9b9f6d72c95026fbaf8721adc36705a5f3f6916387d2c7d15d29f095e4e0/abx_plugins-1.13.103.tar.gz", hash = "sha256:ff49efbee93a54203ecf3680251ef60299f29d4c77b1d20bfbfebfabdccd93b2", size = 460937, upload-time = "2026-09-28T07:58:11.436Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/82/fb926f89622031f8611b942ca9855d42a3158a7640518b2e1d8cac05b4fb/abx_plugins-1.13.82-py3-none-any.whl", hash = "sha256:09a7a7f57beb2b34106765c79ca25d9b27f1dc661d8d16dc2ef6b5bb085c8691", size = 668544, upload-time = "2026-09-27T06:43:51.765Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/69/ec/1c2db942f2ebd8945160084aaed8e4cac29a3364937c64771c8e76700cef/abx_plugins-1.13.103-py3-none-any.whl", hash = "sha256:54ee14dbad090f180b7d35075c175460be0a783745ed86a732bb960157778dff", size = 673577, upload-time = "2026-09-28T07:58:09.949Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
@@ -68,21 +68,21 @@ opencode = [
|
||||
|
||||
[[package]]
|
||||
name = "abxbus"
|
||||
version = "2.5.65"
|
||||
version = "2.5.68"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6b/87/389195fe7d4c864cc1866f8a2b8292ec39c23200fb5f865985d24ad7a56f/abxbus-2.5.65.tar.gz", hash = "sha256:7edbb6036a1c01f13ea5c61f746c54233e709701e53cb674b3159d3727611ef2", size = 134216, upload-time = "2026-09-23T19:18:02.517Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8e/d7/0c92e436230b40ee4c21c83e4081ab67a6930538e3ff759857c3d4cebe51/abxbus-2.5.68.tar.gz", hash = "sha256:e53c68467749f17ba14b982184b02aed8ba106e36412217f1c1cd61d8ce5a7b0", size = 134319, upload-time = "2026-09-28T07:13:42.183Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/21/a1b144425294d71ae6d3874dc27bceb0e62c5fc15d72ca15bab11e0dc724/abxbus-2.5.65-py3-none-any.whl", hash = "sha256:8e9cc083f84ba190e843a3dad13b333bdb857204a2db7e0411d3574d45861131", size = 127915, upload-time = "2026-09-23T19:18:01.182Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/ee/e2b567d576b13bf32aaddcacf376a89c4a4df0de1081f95a56a0d03bdc23/abxbus-2.5.68-py3-none-any.whl", hash = "sha256:61f3ba656f1010b6801d8a625f65f124521e6ec2e03f454baf7491e5ed4fb345", size = 127969, upload-time = "2026-09-28T07:13:40.627Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "abxpkg"
|
||||
version = "1.13.11"
|
||||
version = "1.13.15"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -90,9 +90,9 @@ dependencies = [
|
||||
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ef/6f/b09edc9b41266d7be1433891ac7486dbc656487c77a2e3b210099a18f2ee/abxpkg-1.13.11.tar.gz", hash = "sha256:95a9deb431b8b755e88bcef37f67c69fb045fd0416fe8da70bcfdce7053913d4", size = 242757, upload-time = "2026-09-26T03:08:51.1Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/62/34/16a0501195c9a29c5f532a23a07e50237d0e492b78b5d9d8f43c13a7517c/abxpkg-1.13.15.tar.gz", hash = "sha256:7f29ca4432399ab15180bd58a60f39a3ef7fedcd7940affe2eda6b57e914c98c", size = 242799, upload-time = "2026-09-28T07:26:27.62Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/db/d9/54a2df40e69c451f82e1504e22d272b935a9dd1fe4c3d6d971cac59dab8b/abxpkg-1.13.11-py3-none-any.whl", hash = "sha256:23797f0e6a25012d370517fa8253ef0c898452591e0ad37bfdcd0ea5861c89d3", size = 254167, upload-time = "2026-09-26T03:08:52.578Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7d/85/988013c6e773bed15f1671e017317edc3b047effa594cbf826af49abce11/abxpkg-1.13.15-py3-none-any.whl", hash = "sha256:8b1924225c4f79408d2d1729c59614014779fbae2800d1e7e2ce3ef1389be198", size = 254220, upload-time = "2026-09-28T07:26:26.22Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -127,7 +127,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "archivebox"
|
||||
version = "0.9.71rc3"
|
||||
version = "0.9.71rc42"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
|
||||
@@ -227,10 +227,10 @@ dev = [
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "abx-dl", specifier = "==1.13.59" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.82" },
|
||||
{ name = "abxbus", specifier = "==2.5.65" },
|
||||
{ name = "abxpkg", specifier = "==1.13.11" },
|
||||
{ name = "abx-dl", specifier = "==1.13.71" },
|
||||
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.103" },
|
||||
{ name = "abxbus", specifier = "==2.5.68" },
|
||||
{ name = "abxpkg", specifier = "==1.13.15" },
|
||||
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
|
||||
{ name = "atomicwrites", specifier = "==1.4.1" },
|
||||
{ name = "base32-crockford", specifier = ">=0.3.0" },
|
||||
|
||||
Reference in new issue
Block a user