Compare commits

...
Author SHA1 Message Date
ArchiveBox Release Bot f6360c253f Bump release version to 0.9.71rc42 2026-09-28 08:35:22 +00:00
ArchiveBox Release Bot eaacfa697c Bump for abx-dl 1.13.71 2026-09-28 08:33:18 +00:00
Nick Sweeting 3b99be7fcf Render public pages without waiting for collection totals 2026-09-28 01:32:49 -07:00
ArchiveBox Release Bot 07d0280307 Bump release version to 0.9.71rc41 2026-09-28 08:27:54 +00:00
Nick Sweeting 4fe2b2b7ab Use spare NAS capacity and keep verifier expectations plugin-owned 2026-09-28 01:27:30 -07:00
ArchiveBox Release Bot e5412bfe5a Bump release version to 0.9.71rc40 2026-09-28 08:12:08 +00:00
Nick Sweeting 916c57b42e Merge remote-tracking branch 'origin/dev' into dev 2026-09-28 01:10:04 -07:00
Nick Sweeting 2e6794dd9c Update TLSNotary links to archivebox.io domain 2026-09-28 01:09:59 -07:00
Nick Sweeting 6c67977ad6 Reuse output metadata during public list rendering 2026-09-28 01:09:05 -07:00
ArchiveBox Release Bot 03c6f6a311 Bump release version to 0.9.71rc39 2026-09-28 08:06:52 +00:00
Nick Sweeting 1f710263c5 Link TrueNAS catalog app first in self-hosting instructions 2026-09-28 01:04:56 -07:00
Nick Sweeting cab03ba544 Retain streamed event timings when search latency assertions fail 2026-09-28 01:04:14 -07:00
Nick Sweeting 2d128fdf4b Start extension upload recovery from the existing hook index 2026-09-28 01:03:54 -07:00
Nick Sweeting dc1455ccf3 Require screenshot coverage for every enabled visible preview 2026-09-28 01:03:36 -07:00
ArchiveBox Release Bot 976b6e5b7d Bump release version to 0.9.71rc38 2026-09-28 08:02:45 +00:00
Nick Sweeting 709cd07eab Make live UI and interrupt tests independent of runner DNS and cold installs 2026-09-28 01:00:02 -07:00
ArchiveBox Release Bot 75dcce89dd Bump release version to 0.9.71rc37 2026-09-28 07:55:53 +00:00
Nick Sweeting ae82f09d1a Use indexed public counts and preload snapshot list metadata 2026-09-28 00:55:24 -07:00
Nick Sweeting 8270556ccb Capture plugin-owned screenshot examples through the shared gallery flow 2026-09-28 00:41:42 -07:00
ArchiveBox Release Bot 3502cf4312 Bump release version to 0.9.71rc36 2026-09-28 07:31:16 +00:00
Nick Sweeting bd11f9541f Keep ugNAS runners fed across multiple test waves 2026-09-28 00:31:00 -07:00
ArchiveBox Release Bot 7f326aa57d Bump release version to 0.9.71rc35 2026-09-28 07:01:48 +00:00
Nick Sweeting 1ed926bcd8 Assert status-keyed progress lookups independently of index names 2026-09-28 00:01:00 -07:00
ArchiveBox Release Bot f4c30f5b74 Bump release version to 0.9.71rc34 2026-09-28 06:49:24 +00:00
Nick Sweeting 8362d35549 Merge remote-tracking branch 'origin/dev' into dev 2026-09-27 23:48:29 -07:00
Nick Sweeting a786e79fcd Keep server shutdown cooperative through child reaping 2026-09-27 23:48:16 -07:00
ArchiveBox Release Bot 29fbc9c4df Bump release version to 0.9.71rc33 2026-09-28 06:47:53 +00:00
Nick Sweeting d8e77d5097 Recognize retired dev progress migration during upgrade 2026-09-27 23:46:59 -07:00
ArchiveBox Release Bot 4b301730cd Bump release version to 0.9.71rc32 2026-09-28 06:44:30 +00:00
Nick Sweeting 97061fc849 Merge remote-tracking branch 'origin/dev' into dev 2026-09-27 23:42:06 -07:00
Nick Sweeting 1ea9315492 Keep progress polls on existing indexes and sample crawl totals 2026-09-27 23:41:24 -07:00
ArchiveBox Release Bot 22c30a4f31 Bump release version to 0.9.71rc31 2026-09-28 06:38:57 +00:00
Nick Sweeting da0a6d2221 Remove dev-only progress covering index migration 2026-09-27 23:38:08 -07:00
Nick Sweeting 9315887f1d Remove unreachable foreground shutdown branches and explain signal boundaries 2026-09-27 23:34:23 -07:00
ArchiveBox Release Bot d4faa4a034 Bump release version to 0.9.71rc30 2026-09-28 06:24:34 +00:00
Nick Sweeting dc4d75ff2f Send shutdown regression signals from an independent process 2026-09-27 23:24:12 -07:00
ArchiveBox Release Bot 00de9bbcbb Bump release version to 0.9.71rc29 2026-09-28 06:12:24 +00:00
Nick Sweeting 5b25b39f7a Exercise real shutdown signals while polling child processes 2026-09-27 23:12:08 -07:00
ArchiveBox Release Bot 5e069e6e1f Bump release version to 0.9.71rc28 2026-09-28 02:24:18 +00:00
Nick Sweeting acabe3fbd6 Capture Popen lock state when killed child cannot be reaped 2026-09-27 19:18:53 -07:00
ArchiveBox Release Bot d0eb7c60e5 Bump release version to 0.9.71rc27 2026-09-28 02:17:47 +00:00
ArchiveBox Release Bot 7f0298e1f4 Bump for abx-dl 1.13.70 2026-09-28 02:17:29 +00:00
ArchiveBox Release Bot 4ee4613627 Bump release version to 0.9.71rc26 2026-09-28 02:03:56 +00:00
ArchiveBox Release Bot 489f0a1b64 Bump for abx-dl 1.13.69 2026-09-28 02:03:38 +00:00
ArchiveBox Release Bot 1045a0e7e8 Bump release version to 0.9.71rc25 2026-09-28 01:34:05 +00:00
ArchiveBox Release Bot 93b70398e3 Bump for abx-dl 1.13.68 2026-09-28 01:33:50 +00:00
ArchiveBox Release Bot 5992353dca Bump release version to 0.9.71rc24 2026-09-28 00:33:26 +00:00
Nick Sweeting fa3af93cf0 Preserve shutdown diagnostics and always clean up descendants 2026-09-27 17:32:01 -07:00
ArchiveBox Release Bot 05a43c2c9d Bump release version to 0.9.71rc23 2026-09-28 00:27:34 +00:00
ArchiveBox Release Bot 33797e3d6a Bump for abx-dl 1.13.66 2026-09-28 00:26:54 +00:00
ArchiveBox Release Bot cc6e208171 Bump release version to 0.9.71rc22 2026-09-28 00:16:28 +00:00
Nick Sweeting b6cfe542a5 Capture gallery viewports through one browser path 2026-09-27 17:16:09 -07:00
ArchiveBox Release Bot ac6ce915b1 Bump release version to 0.9.71rc21 2026-09-27 23:52:29 +00:00
Nick Sweeting c7a766fb98 Exercise stable reconciliation through the shared preparation workflow 2026-09-27 16:50:04 -07:00
ArchiveBox Release Bot cdb0a3b582 Bump release version to 0.9.71rc20 2026-09-27 23:42:36 +00:00
ArchiveBox Release Bot 9a85d8dfa2 Bump for abx-dl 1.13.63 2026-09-27 23:41:33 +00:00
ArchiveBox Release Bot 34e4211383 Bump release version to 0.9.71rc19 2026-09-27 23:39:00 +00:00
Nick Sweeting 43edc66452 Normalize the shared release workflow file ending 2026-09-27 16:38:30 -07:00
ArchiveBox Release Bot 5f44ca29a9 Bump release version to 0.9.71rc18 2026-09-27 23:35:50 +00:00
Nick Sweeting 048fd4113a Share the version gate before CI and screenshot captures 2026-09-27 16:23:32 -07:00
ArchiveBox Release Bot e7da938f0b Bump release version to 0.9.71rc17 2026-09-27 23:18:39 +00:00
Nick Sweeting f4123e80c9 Validate saved binary paths once when loading a crawl 2026-09-27 16:17:23 -07:00
Nick Sweeting b035561b85 Keep unrelated host usage out of per-snapshot memory cost 2026-09-27 16:16:01 -07:00
Nick Sweeting 86c30d955c Register CLI machines explicitly and repair migrated process links 2026-09-27 16:16:01 -07:00
ArchiveBox Release Bot f9a0cab649 Bump release version to 0.9.71rc16 2026-09-27 22:59:22 +00:00
Nick Sweeting 813a93109b Keep live progress polling read-only and use indexed queries 2026-09-27 15:57:31 -07:00
Nick Sweeting 19c3239bad Avoid waiting for absent progress panels in gallery captures 2026-09-27 15:57:28 -07:00
ArchiveBox Release Bot 1b795c6c07 Bump release version to 0.9.71rc15 2026-09-27 22:50:12 +00:00
Nick Sweeting b9bad0abae Apply abxpkg environment deltas before launching test hooks 2026-09-27 15:48:34 -07:00
ArchiveBox Release Bot 1c0335c131 Bump release version to 0.9.71rc14 2026-09-27 22:38:42 +00:00
Nick Sweeting 861eb0f607 Use the existing HTTP fixture for interrupt rendering coverage 2026-09-27 15:37:58 -07:00
ArchiveBox Release Bot 5f8bd00a69 Bump release version to 0.9.71rc13 2026-09-27 22:23:22 +00:00
Nick Sweeting c20b875736 Schedule measured long tests first across both runner pools 2026-09-27 15:23:02 -07:00
ArchiveBox Release Bot 5e8bea8d6a Bump release version to 0.9.71rc12 2026-09-27 21:50:39 +00:00
Nick Sweeting b167d889b5 Require live progress and real screencast content in marketing screenshots 2026-09-27 14:48:36 -07:00
ArchiveBox Release Bot 5a6d78566d Bump release version to 0.9.71rc11 2026-09-27 21:41:09 +00:00
ArchiveBox Release Bot 25cb823a43 Bump for abx-dl 1.13.62 2026-09-27 21:40:17 +00:00
ArchiveBox Release Bot c7402195e4 Bump release version to 0.9.71rc10 2026-09-27 21:16:33 +00:00
Nick Sweeting 29a941bd9f Prepare versions before CI fanout and remove recursive documentation checks 2026-09-27 14:15:46 -07:00
ArchiveBox Release Bot 224da46da5 Bump release version to 0.9.71rc9 2026-09-27 21:14:37 +00:00
ArchiveBox Release Bot 396ba92672 Bump for abx-dl 1.13.60 2026-09-27 21:14:20 +00:00
ArchiveBox Release Bot 4ff3aa3061 Bump release version to 0.9.71rc8 2026-09-27 21:12:49 +00:00
Nick Sweeting 62b6cf80da Fix link to Screenshots in README
Updated the link to the Screenshots section in the README.
2026-09-27 17:12:01 -04:00
ArchiveBox Release Bot 5fe08ce366 Bump release version to 0.9.71rc7 2026-09-27 21:01:59 +00:00
Nick Sweeting b2d6c5a99a Fix long tags and responsive snapshot card layout 2026-09-27 14:01:37 -07:00
ArchiveBox Release Bot bfb36aee49 Bump release version to 0.9.71rc6 2026-09-27 20:59:22 +00:00
Nick Sweeting 04a9c98e6a Keep exact image validation when CI pushes release digests 2026-09-27 13:58:52 -07:00
ArchiveBox Release Bot 815bf05e07 Bump release version to 0.9.71rc5 2026-09-27 20:56:29 +00:00
Nick Sweeting 1e3623fee3 Split long test files and publish artifacts already accepted by CI 2026-09-27 13:56:07 -07:00
ArchiveBox Release Bot 6c83d7e23c Bump release version to 0.9.71rc4 2026-09-27 20:19:40 +00:00
Nick Sweeting c1af5599a5 Reduce repeated CI setup and migration work without dropping coverage 2026-09-27 13:12:50 -07:00
98 changed files with 9814 additions and 8285 deletions

No files matched your search

+1 -1
View File
@@ -10,4 +10,4 @@ Also review the inline navbar/footer in abx-dl/website/index.html and
-->
<footer class="abx-footer" aria-label="ArchiveBox ecosystem">
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg class="abx-logo" viewBox="0 0 40 40" aria-hidden="true"><rect width="40" height="40" rx="10" fill="#9b2854"/><path d="M10 14h20v17H10z" fill="#fff4ed"/><path d="M8 9h24v6H8z" fill="#fff4ed"/><path d="M16 19h8v3h-8z" fill="#9b2854"/><path d="M12 31V17" stroke="#9b2854" stroke-width="2"/></svg><span>ArchiveBox</span></a><p class="abx-tagline">Open-source tools for keeping a lasting copy of the web.<br>Your data. Your devices. Your archive.</p></div><p class="abx-footer-motto">A home for the web<br>you want to keep.</p></div>
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Apps &amp; tools</h2><ul><li><a href="https://archivebox.io/">ArchiveBox Server</a><a class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" aria-label="ArchiveBox Server source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox for macOS &amp; iOS</a><a class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" aria-label="ArchiveBox for macOS &amp; iOS source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox for Android</a><a class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" aria-label="ArchiveBox for Android source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox Desktop</a><a class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" aria-label="ArchiveBox Desktop source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox Browser Extension</a><a class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" aria-label="ArchiveBox Browser Extension source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox Plugin Library</a><a class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" aria-label="ArchiveBox Plugin Library source on GitHub" title="View source on GitHub"><svg viewBox="0 0 24 24" aria-hidden="true"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"/></svg></a></li><li><a href="https://abx-dl.archivebox.io/">One-shot abx-Line truncated
+43 -8
View File
@@ -1,30 +1,65 @@
#!/usr/bin/env python3
"""Discover every ArchiveBox test file and expose it as one CI job."""
"""Discover every test once, batching measured short files."""
import json
import os
from pathlib import Path
def main() -> None:
root = Path.cwd().resolve()
root = Path(__file__).resolve().parents[2]
archivebox_tests = sorted((root / "archivebox/tests").glob("test_*.py"))
if not archivebox_tests:
raise SystemExit("No ArchiveBox tests discovered")
matrix: list[dict[str, object]] = []
# These are observed whole-job seconds, including setup. Unknown/long files,
# optional dependencies and hosted-only requirements keep independent jobs.
durations = json.loads((root / ".github/test-durations.json").read_text())["seconds"]
hosted = {path for path in archivebox_tests if "# ci-runner: hosted" in path.read_text().splitlines()[:5]}
requested = os.environ.get("TEST_FILE", "")
batches: list[list[str]] = []
batch: list[str] = []
batch_seconds = 0
for path in archivebox_tests:
test_path = path.relative_to(root).as_posix()
duration = durations.get(test_path, 60)
if duration >= 60 or path in hosted or path.stem == "test_auth_ldap" or test_path == requested:
batches.append([test_path])
continue
if batch and (len(batch) == 8 or batch_seconds + duration > 180):
batches.append(batch)
batch = []
batch_seconds = 0
batch.append(test_path)
batch_seconds += duration
if batch:
batches.append(batch)
matrix: list[dict[str, object]] = []
for paths in batches:
matrix.append(
{
"name": f"main/{path.stem}",
"paths": [test_path],
"paths_arg": test_path,
"extra": "ldap" if test_path.endswith("/test_auth_ldap.py") else "",
"count": 1,
"name": f"main/{Path(paths[0]).stem}" if len(paths) == 1 else f"batch/{Path(paths[0]).stem}",
"paths": paths,
"paths_arg": " ".join(paths),
"extra": "ldap" if paths == ["archivebox/tests/test_auth_ldap.py"] else "",
"count": len(paths),
"ugnas": False,
},
)
# Ordinary Linux tests can use either pool. Only genuine hosted-only
# requirements need a file header; filenames never enter runner policy.
# Keep a bounded NAS share so hosted capacity continues working in parallel.
capacity = int(os.environ.get("UGNAS_CI_MAX_JOBS", "3"))
# Start long jobs first in both pools instead of leaving several minutes of
# work behind every short job in an alphabetically ordered hosted queue.
matrix.sort(key=lambda item: sum(durations.get(path, 60) for path in item["paths"]), reverse=True)
eligible = [item for item in matrix if not any(root / path in hosted for path in item["paths"])]
for item in eligible[:capacity]:
item["ugnas"] = True
discovered_paths = [path for entry in matrix for path in entry["paths"]]
if len(discovered_paths) != len(set(discovered_paths)):
raise SystemExit("Tests were not discovered exactly once")
+168
View File
@@ -0,0 +1,168 @@
{
"source_runs": [
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/36302511923",
"https://github.com/ArchiveBox/ArchiveBox/actions/runs/36353196552"
],
"measurement": "Whole job seconds, including setup and excluding queue time. Unbatched files use the latest complete successful run; batched files retain their individual baseline measurement. Unknown files stay independent.",
"seconds": {
"archivebox/tests/test_api_v1_auth_browser_session.py": 29,
"archivebox/tests/test_api_v1_auth_check_api_token.py": 27,
"archivebox/tests/test_api_v1_auth_get_api_token.py": 24,
"archivebox/tests/test_api_v1_cli_add.py": 96,
"archivebox/tests/test_api_v1_cli_remove.py": 50,
"archivebox/tests/test_api_v1_cli_schedule.py": 41,
"archivebox/tests/test_api_v1_cli_search.py": 26,
"archivebox/tests/test_api_v1_cli_update.py": 100,
"archivebox/tests/test_api_v1_cli_workflow_add_search_update_remove.py": 64,
"archivebox/tests/test_api_v1_core_any_id.py": 21,
"archivebox/tests/test_api_v1_core_archiveresult_archiveresult_id.py": 55,
"archivebox/tests/test_api_v1_core_archiveresults.py": 32,
"archivebox/tests/test_api_v1_core_snapshot_snapshot_id.py": 203,
"archivebox/tests/test_api_v1_core_snapshots_rss.py": 27,
"archivebox/tests/test_api_v1_core_snapshots.py": 28,
"archivebox/tests/test_api_v1_core_tag_tag_id_rename.py": 23,
"archivebox/tests/test_api_v1_core_tag_tag_id_snapshots_jsonl.py": 27,
"archivebox/tests/test_api_v1_core_tag_tag_id_urls_txt.py": 28,
"archivebox/tests/test_api_v1_core_tag_tag_id.py": 27,
"archivebox/tests/test_api_v1_core_tags_add_to_snapshot.py": 27,
"archivebox/tests/test_api_v1_core_tags_autocomplete.py": 26,
"archivebox/tests/test_api_v1_core_tags_create.py": 21,
"archivebox/tests/test_api_v1_core_tags_remove_from_snapshot.py": 25,
"archivebox/tests/test_api_v1_core_tags_search.py": 26,
"archivebox/tests/test_api_v1_core_tags.py": 25,
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_filename.py": 22,
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_filename.py": 25,
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id_files_folder_subfolder_filename.py": 24,
"archivebox/tests/test_api_v1_crawls_crawl_crawl_id.py": 86,
"archivebox/tests/test_api_v1_crawls_crawls.py": 21,
"archivebox/tests/test_api_v1_machine_binaries.py": 23,
"archivebox/tests/test_api_v1_machine_binary_binary_id.py": 23,
"archivebox/tests/test_api_v1_machine_binary_by_name_name.py": 26,
"archivebox/tests/test_api_v1_machine_machine_current.py": 23,
"archivebox/tests/test_api_v1_machine_machine_machine_id.py": 23,
"archivebox/tests/test_api_v1_machine_machines.py": 27,
"archivebox/tests/test_api_v1_personas_personas.py": 23,
"archivebox/tests/test_api_v1_personas_sync.py": 29,
"archivebox/tests/test_api_v1_workflow_core_token_auth_side_effects.py": 36,
"archivebox/tests/test_api_v1_workflow_frozen_crawl_config_sources.py": 33,
"archivebox/tests/test_archive_result_service.py": 122,
"archivebox/tests/test_auth_ldap.py": 44,
"archivebox/tests/test_binary_service.py": 83,
"archivebox/tests/test_cli_add_1.py": 179,
"archivebox/tests/test_cli_add_2.py": 167,
"archivebox/tests/test_cli_add_3.py": 194,
"archivebox/tests/test_cli_add_4.py": 168,
"archivebox/tests/test_cli_archiveresult.py": 145,
"archivebox/tests/test_cli_binary.py": 20,
"archivebox/tests/test_cli_config.py": 141,
"archivebox/tests/test_cli_crawl.py": 127,
"archivebox/tests/test_cli_extract_input.py": 205,
"archivebox/tests/test_cli_extract.py": 113,
"archivebox/tests/test_cli_help.py": 30,
"archivebox/tests/test_cli_init.py": 133,
"archivebox/tests/test_cli_install.py": 58,
"archivebox/tests/test_cli_list.py": 177,
"archivebox/tests/test_cli_machine.py": 50,
"archivebox/tests/test_cli_manage.py": 48,
"archivebox/tests/test_cli_mcp.py": 51,
"archivebox/tests/test_cli_oneshot.py": 23,
"archivebox/tests/test_cli_persona.py": 82,
"archivebox/tests/test_cli_piping.py": 97,
"archivebox/tests/test_cli_pluginmap.py": 21,
"archivebox/tests/test_cli_process.py": 20,
"archivebox/tests/test_cli_remove.py": 150,
"archivebox/tests/test_cli_run_1.py": 165,
"archivebox/tests/test_cli_run_2.py": 241,
"archivebox/tests/test_cli_run_3.py": 162,
"archivebox/tests/test_cli_run_4.py": 200,
"archivebox/tests/test_cli_schedule.py": 87,
"archivebox/tests/test_cli_search.py": 35,
"archivebox/tests/test_cli_server.py": 251,
"archivebox/tests/test_cli_shell.py": 32,
"archivebox/tests/test_cli_snapshot.py": 138,
"archivebox/tests/test_cli_status.py": 105,
"archivebox/tests/test_cli_tag.py": 21,
"archivebox/tests/test_cli_update_reindex_snapshots.py": 144,
"archivebox/tests/test_cli_update.py": 148,
"archivebox/tests/test_cli_version.py": 181,
"archivebox/tests/test_config_DELETE_AFTER.py": 192,
"archivebox/tests/test_config_MAX_limits.py": 34,
"archivebox/tests/test_config_ONLY_NEW.py": 24,
"archivebox/tests/test_config_SAVE_TITLE.py": 99,
"archivebox/tests/test_config_URL_filters.py": 26,
"archivebox/tests/test_core_config.py": 20,
"archivebox/tests/test_core_forms.py": 28,
"archivebox/tests/test_crawl_pause.py": 20,
"archivebox/tests/test_crawl_runner.py": 45,
"archivebox/tests/test_crawl_service.py": 34,
"archivebox/tests/test_frozen_crawl_config.py": 47,
"archivebox/tests/test_hook_result_lifecycle.py": 78,
"archivebox/tests/test_hooks.py": 90,
"archivebox/tests/test_machine_models.py": 52,
"archivebox/tests/test_machine_service.py": 104,
"archivebox/tests/test_migrations_04_to_09.py": 36,
"archivebox/tests/test_migrations_07_to_09.py": 140,
"archivebox/tests/test_migrations_08_to_09.py": 257,
"archivebox/tests/test_migrations_fresh.py": 99,
"archivebox/tests/test_misc_checks.py": 22,
"archivebox/tests/test_misc_monkey_patches.py": 20,
"archivebox/tests/test_opencode_agent_1.py": 135,
"archivebox/tests/test_opencode_agent_2.py": 138,
"archivebox/tests/test_opencode_agent_3.py": 140,
"archivebox/tests/test_opencode_browser_1.py": 290,
"archivebox/tests/test_opencode_browser_2.py": 157,
"archivebox/tests/test_opencode_browser_3.py": 129,
"archivebox/tests/test_orm_helpers.py": 22,
"archivebox/tests/test_permissions.py": 21,
"archivebox/tests/test_persona_runtime.py": 56,
"archivebox/tests/test_postgres_backend.py": 85,
"archivebox/tests/test_process_runtime_paths.py": 28,
"archivebox/tests/test_process_service.py": 49,
"archivebox/tests/test_proof_previews.py": 54,
"archivebox/tests/test_recursive_crawl_1.py": 396,
"archivebox/tests/test_recursive_crawl_2.py": 193,
"archivebox/tests/test_recursive_crawl_3.py": 144,
"archivebox/tests/test_release_notes.py": 14,
"archivebox/tests/test_release_source_fingerprint.py": 22,
"archivebox/tests/test_release_workflow.py": 21,
"archivebox/tests/test_resource_admission.py": 21,
"archivebox/tests/test_runner_watch.py": 182,
"archivebox/tests/test_search.py": 203,
"archivebox/tests/test_serve_static.py": 31,
"archivebox/tests/test_server_automatic_url.py": 36,
"archivebox/tests/test_server_security_browser.py": 241,
"archivebox/tests/test_settings_signal_webhooks.py": 21,
"archivebox/tests/test_setup_script.py": 21,
"archivebox/tests/test_shutdown_util.py": 22,
"archivebox/tests/test_snapshot_filesystem_migration.py": 29,
"archivebox/tests/test_snapshot_output_stacks.py": 62,
"archivebox/tests/test_snapshot_ref_lookup.py": 23,
"archivebox/tests/test_snapshot_service.py": 41,
"archivebox/tests/test_snapshot_url_length.py": 26,
"archivebox/tests/test_tag_service.py": 24,
"archivebox/tests/test_takeover_util_1.py": 122,
"archivebox/tests/test_takeover_util_2.py": 171,
"archivebox/tests/test_takeover_util_3.py": 165,
"archivebox/tests/test_takeover_util_4.py": 160,
"archivebox/tests/test_test_harness.py": 22,
"archivebox/tests/test_ui_add_view_runtime.py": 239,
"archivebox/tests/test_ui_add_view.py": 46,
"archivebox/tests/test_ui_admin_apitoken.py": 23,
"archivebox/tests/test_ui_admin_archiveresult.py": 30,
"archivebox/tests/test_ui_admin_config_widget.py": 25,
"archivebox/tests/test_ui_admin_crawl.py": 44,
"archivebox/tests/test_ui_admin_links.py": 35,
"archivebox/tests/test_ui_admin_machine.py": 29,
"archivebox/tests/test_ui_admin_persona.py": 28,
"archivebox/tests/test_ui_admin_snapshot.py": 288,
"archivebox/tests/test_ui_admin_tag.py": 26,
"archivebox/tests/test_ui_admin_user.py": 28,
"archivebox/tests/test_ui_config_views.py": 43,
"archivebox/tests/test_ui_live_progress.py": 303,
"archivebox/tests/test_ui_public_snapshot.py": 138,
"archivebox/tests/test_ui_savepagenow.py": 48,
"archivebox/tests/test_urls.py": 179,
"archivebox/tests/test_util.py": 20,
"archivebox/tests/test_version_metadata.py": 24
}
}
+23 -2
View File
@@ -18,47 +18,68 @@ concurrency:
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }}
jobs:
prepare:
permissions:
contents: write
uses: ./.github/workflows/prepare-release.yml
secrets: inherit
lint:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
name: Linters
uses: ./.github/workflows/lint.yml
secrets: inherit
install-cli-platform:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
name: Install and CLI platform compatibility
uses: ./.github/workflows/test.yml
secrets: inherit
tests:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
name: Discovered test matrix
uses: ./.github/workflows/test-parallel.yml
secrets: inherit
documentation:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
name: Documentation and root tests
uses: ./.github/workflows/docs.yml
secrets: inherit
codeql:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
name: CodeQL
uses: ./.github/workflows/codeql.yml
secrets: inherit
python-artifacts:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
name: Tested Python artifacts
uses: ./.github/workflows/pip.yml
secrets: inherit
docker-digests:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
name: Tested Docker images
uses: ./.github/workflows/docker.yml
with:
push_digests: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
push_digests: ${{ github.event_name == 'push' && (github.ref_name == 'dev' || github.ref_name == 'main') }}
secrets: inherit
required:
name: All required CI lanes
if: ${{ always() }}
if: ${{ always() && needs.prepare.outputs.run_jobs == 'true' }}
needs:
- prepare
- lint
- install-cli-platform
- tests
+2 -2
View File
@@ -65,7 +65,7 @@ jobs:
if: github.event_name == 'workflow_dispatch'
shell: bash
env:
SOURCE_REF: ${{ inputs.source_ref }}
SOURCE_REF: ${{ inputs.source_ref || github.sha }}
run: |
set -Eeuo pipefail
[[ "$SOURCE_REF" =~ ^[0-9a-f]{40}$ ]]
@@ -295,7 +295,7 @@ jobs:
'
- name: Validate exact built image version and commit
if: ${{ !inputs.push_digests }}
if: inputs.full_tests
shell: bash
run: |
set -Eeuo pipefail
+1 -1
View File
@@ -67,7 +67,7 @@ jobs:
timeout-minutes: 15
strategy:
fail-fast: false
max-parallel: 4
max-parallel: 8
matrix: ${{ fromJSON(needs.discover.outputs.matrix) }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -1,32 +1,33 @@
name: Release candidate
name: Prepare release version
on:
push:
branches: [dev, main]
workflow_call:
outputs:
run_jobs:
description: Whether this revision owns its release version and should run expensive jobs
value: ${{ jobs.prepare.outputs.run_jobs }}
permissions:
actions: read
contents: write
packages: write
concurrency:
group: release-candidate-${{ github.ref_name }}
cancel-in-progress: true
jobs:
prepare:
name: Prepare the version before starting expensive jobs
permissions:
contents: write
runs-on: ubuntu-24.04
outputs:
action: ${{ steps.version.outputs.action }}
run_jobs: ${{ github.event_name != 'push' || steps.channel.outputs.stable == 'true' || steps.version.outputs.action == 'ready' || steps.version.outputs.action == 'reserved' }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
submodules: true
token: ${{ secrets.RELEASE_GH_TOKEN }}
token: ${{ github.event_name == 'push' && secrets.RELEASE_GH_TOKEN || github.token }}
- name: Keep synchronized stable branches on the released version
id: channel
if: github.event_name == 'push'
shell: bash
run: |
set -Eeuo pipefail
@@ -45,41 +46,10 @@ jobs:
fi
- id: version
if: steps.channel.outputs.stable != 'true'
if: github.event_name == 'push' && steps.channel.outputs.stable != 'true'
uses: ArchiveBox/monorepo/.github/actions/prepare-release-version@main
with:
package: archivebox
branch: ${{ github.ref_name }}
tag-prefix: v
version-scheme: ${{ github.ref_name == 'main' && 'patch' || 'rc' }}
python-artifacts:
needs: prepare
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
uses: ./.github/workflows/pip.yml
with:
full_tests: false
secrets: inherit
docker-digests:
needs: prepare
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
uses: ./.github/workflows/docker.yml
with:
push_digests: true
full_tests: false
secrets: inherit
opencode:
needs: prepare
if: github.ref_name == 'dev' && (needs.prepare.outputs.action == 'ready' || needs.prepare.outputs.action == 'reserved')
strategy:
fail-fast: false
matrix:
test_file:
- archivebox/tests/test_opencode_agent.py
- archivebox/tests/test_opencode_browser.py
uses: ./.github/workflows/test-parallel.yml
with:
test_file: ${{ matrix.test_file }}
secrets: inherit
+4 -16
View File
@@ -2,7 +2,7 @@ name: Release
on:
workflow_run:
workflows: [CI, Release candidate]
workflows: [CI]
types: [completed]
branches: [dev, main]
workflow_dispatch:
@@ -111,26 +111,14 @@ jobs:
--jq '[.status, .conclusion, .headSha, .event, .name] | @tsv')"
[[ "$CI_STATUS" == "$(printf 'completed\tsuccess\t%s\tpush\tCI' "$RELEASE_SHA")" ]]
else
CANDIDATE_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow release-candidate.yml \
--commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
--json databaseId,status,conclusion \
--jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
CI_RUN_ID="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml \
--commit "$RELEASE_SHA" --branch "$RELEASE_BRANCH" --event push --limit 1 \
--json databaseId,status,conclusion \
--jq '.[0] | select(.status == "completed" and .conclusion == "success") | .databaseId // empty')"
[[ -n "$CANDIDATE_RUN_ID" && -n "$CI_RUN_ID" ]] || exit 0
if [[ "$RELEASE_BRANCH" == dev ]]; then
DIGEST_RUN_ID="$CANDIDATE_RUN_ID"
else
DIGEST_RUN_ID="$CI_RUN_ID"
fi
fi
if [[ "$RELEASE_BRANCH" == dev && "$GITHUB_EVENT_NAME" != workflow_dispatch ]]; then
ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"
else
ARTIFACT_RUN_ID="$CI_RUN_ID"
[[ -n "$CI_RUN_ID" ]] || exit 0
DIGEST_RUN_ID="$CI_RUN_ID"
fi
ARTIFACT_RUN_ID="$CI_RUN_ID"
{
echo 'ready=true'
echo "artifact_run_id=$ARTIFACT_RUN_ID"
+9
View File
@@ -13,6 +13,7 @@ on:
- pyproject.toml
- uv.lock
- .github/workflows/screenshots.yml
- .github/workflows/prepare-release.yml
workflow_dispatch:
permissions:
@@ -23,7 +24,15 @@ concurrency:
cancel-in-progress: false
jobs:
prepare:
permissions:
contents: write
uses: ./.github/workflows/prepare-release.yml
secrets: inherit
deploy:
needs: prepare
if: needs.prepare.outputs.run_jobs == 'true'
runs-on: ubuntu-24.04
steps:
- name: Checkout
+30 -5
View File
@@ -42,7 +42,9 @@ jobs:
organization: ArchiveBox
primary-runner: self-hosted,ugnas-linux
fallback-runner: ubuntu-24.04
primaries-required: 1
# Jobs queued for this pool must not be mistaken for idle runner slots:
# requiring MAX_JOBS idle runners prevents assigning a second wave.
primaries-required: ${{ vars.UGNAS_CI_MIN_IDLE || 1 }}
fallback-on-error: true
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
@@ -87,6 +89,7 @@ jobs:
id: discover
env:
TEST_FILE: ${{ inputs.test_file }}
UGNAS_CI_MAX_JOBS: ${{ vars.UGNAS_CI_MAX_JOBS || 3 }}
run: |
set -Eeuo pipefail
mapfile -t output < <("$PYTHON_BINARY" .github/scripts/discover_test_matrix.py)
@@ -102,10 +105,10 @@ jobs:
name: ${{ matrix.test.name }}
needs: discover-tests
# Opt in trusted branches only; PRs always use GitHub-hosted runners.
# UGNAS_CI_TESTS is a JSON array of matrix names, e.g. ["main/test_cli_list"].
# Ordinary Linux tests are eligible; # ci-runner: hosted declares exceptions.
# The hosted discovery job falls back when workers are offline/busy or the
# availability API fails. UGNAS_CI_ENABLED=false disables the pool entirely.
runs-on: ${{ fromJSON(contains(fromJSON(vars.UGNAS_CI_TESTS || '[]'), matrix.test.name) && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
runs-on: ${{ fromJSON(matrix.test.ugnas && needs.discover-tests.outputs.runner || '["ubuntu-24.04"]') }}
env:
CHROME_HEADLESS: "true"
@@ -186,13 +189,35 @@ jobs:
for test_path in "${test_paths[@]}"; do
test -f "$test_path"
done
test_status=0
for test_path in "${test_paths[@]}"; do
test_name="$(basename "$test_path" .py)"
test_slug="${test_name//[^A-Za-z0-9_.-]/_}"
export DATA_DIR="$RUNNER_TEMP/archivebox-test-data/$test_slug"
export PERSONAS_DIR="$RUNNER_TEMP/abx-personas/$test_slug"
export CHROME_USER_DATA_DIR="$PERSONAS_DIR/Default/chrome_profile"
if (( ${#test_paths[@]} > 1 )); then
export ABXPKG_LIB_DIR="$RUNNER_TEMP/archivebox-lib/$test_slug"
fi
mkdir -p "$DATA_DIR" "$PERSONAS_DIR" "$RUNNER_TEMP/pytest/$test_slug"
uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" \
--basetemp="$RUNNER_TEMP/pytest/$test_slug"
uv run --no-cache --no-sync --no-sources pytest -vs "$test_path" --durations=20 \
--basetemp="$RUNNER_TEMP/pytest/$test_slug" \
--junitxml="$RUNNER_TEMP/test-results/$test_slug.xml" || test_status=1
done
exit "$test_status"
- name: Save test results
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: test-results-${{ strategy.job-index }}
path: ${{ runner.temp }}/test-results/*.xml
if-no-files-found: error
retention-days: 14
- name: Save failed runtime diagnostics
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: runtime-logs-${{ strategy.job-index }}
path: ${{ runner.temp }}/pytest/**/*.log
if-no-files-found: ignore
retention-days: 14
+2 -6
View File
@@ -77,11 +77,7 @@ archivebox run
## Verification
Use targeted tests for focused work:
```bash
uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
uv run prek run --all-files
```
Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
with `uv run prek run --all-files`. For focused work, select the relevant tests.
Releases are published only by `.github/workflows/release.yml` after complete CI succeeds for the exact commit. Pushes to `main` prepare stable patch releases automatically; `dev` prepares release candidates. Version selection and dependency repinning are owned by CI and the monorepo coordinator. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
+2 -3
View File
@@ -105,12 +105,11 @@ curl -fsSL 'https://get.archivebox.io' | bash
</details>
<br/>
<div align="center" style="text-align: center">
<br/><br/>
<img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic"> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/b2765a33-0d1e-4019-a1db-920c7e00e20e" height="75px" alt="logo" align="top"/> &nbsp; <img src="https://github.com/ArchiveBox/ArchiveBox/assets/511499/5a7d95f2-6977-4de6-9f08-42851a1fe1d2" height="70px" alt="bookshelf graphic">
<br/><br/>
<small><a href="https://demo.archivebox.io">Demo</a> | <a href="#screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
<small><a href="https://demo.archivebox.io">Demo</a> | <a href="https://archivebox.io/screenshots">Screenshots</a> | <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Usage">Usage</a></small>
<br/>
<sub>. . . . . . . . . . . . . . . . . . . . . . . . . . . .</sub>
<br/><br/>
@@ -381,7 +380,7 @@ See <a href="#%EF%B8%8F-cli-usage">below</a> for usage examples using the CLI, W
> *Warning: These are contributed by external volunteers and may lag behind the official Docker and `uv` channels.*
<ul>
<li><s>TrueNAS: <a href="https://truecharts.org/charts/stable/archivebox/">Official ArchiveBox TrueChart</a> / <a href="https://dev.to/finloop/setting-up-archivebox-on-truenas-scale-1788">Custom App Guide</a></s> (<a href="https://truecharts.org/news/scale-deprecation/">TrueCharts is discontinued</a>, wait for <a href="https://forums.truenas.com/t/the-future-of-electric-eel-and-apps/5409/">Electric Eel</a>)</li>
<li><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a></li>
<li><a href="https://unraid.net/community/apps?q=archivebox#r">UnRaid</a></li>
<li><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a></li>
<li><a href="https://github.com/YunoHost-Apps/archivebox_ynh">Yunohost</a></li>
+8
View File
@@ -182,6 +182,14 @@ def crawl_file(request: HttpRequest, crawl_id: str, path: str):
if not file_path.is_file() or crawl_root not in file_path.parents:
raise HttpError(404, "Crawl file not found")
if path == "chrome_screencast/latest.jpg" and request.GET.get("after"):
try:
started_at = float(request.GET["after"])
except ValueError:
raise HttpError(400, "Invalid screencast start time")
if file_path.stat().st_mtime < started_at:
raise HttpError(404, "No frame for this capture yet")
response = FileResponse(file_path.open("rb"))
response["Cache-Control"] = "no-store, no-cache, max-age=0, must-revalidate"
response["Pragma"] = "no-cache"
+6 -2
View File
@@ -174,8 +174,12 @@ def cli(ctx, help=False):
archivebox_init(quick=True)
os.environ.pop("ARCHIVEBOX_WANTS_INIT", None)
check_data_folder()
if subcommand != "update":
check_migrations(auto_apply=True)
if subcommand != "update" and not check_migrations(auto_apply=True):
from archivebox.machine.models import Machine
# Host registration and config reconciliation belong to CLI
# startup; get_config() is also used by read-only HTTP views.
Machine.current()
except Exception as e:
STDERR.print(f"[red][X] Error setting up Django or checking data folder: {e}[/red]")
if subcommand not in ("manage", "shell"): # not all management commands need django to be setup beforehand
+5
View File
@@ -111,6 +111,11 @@ def init(force: bool = False, quick: bool = False, install: bool = False) -> Non
else:
os.environ["ARCHIVEBOX_WANTS_INIT"] = previous_wants_init
from archivebox.machine.models import Machine
# Registration requires the final schema and reconciles legacy file config.
Machine.current()
from archivebox.misc.db import database_display_location, is_postgres
assert database_exists()
+1 -1
View File
@@ -1113,7 +1113,7 @@ def get_config(
if apps.ready:
from archivebox.machine.models import Machine
machine = Machine.current()
machine = Machine.current_readonly()
except (DatabaseError, ImportError, RuntimeError, TypeError, ValueError):
machine = None
@@ -0,0 +1,25 @@
"""Repair process links after both core's legacy copy and machine migrations."""
from importlib import import_module
from django.db import migrations
repair_process_binary_iface_links = import_module(
"archivebox.machine.migrations.0020_repair_process_binary_iface_links",
).repair_process_binary_iface_links
class Migration(migrations.Migration):
atomic = False
dependencies = [
("core", "0055_repair_legacy_retry_timestamps"),
("machine", "0022_networkinterface_identity_without_mac"),
]
operations = [
# The original repair can run before core.0027 creates legacy Process
# rows. Repair again after the copy, including already upgraded DBs.
migrations.RunPython(repair_process_binary_iface_links, migrations.RunPython.noop),
]
+21 -9
View File
@@ -4436,6 +4436,10 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
return 0
def output_file_map(self) -> dict[str, dict[str, Any]]:
# List views opt into this cache on their read-only, request-local rows.
# Runner-owned results must still reflect metadata changed during a hook.
if "_render_output_file_map" in self.__dict__:
return self._render_output_file_map
return self._normalize_output_files(self.output_files)
def output_file_paths(self) -> list[str]:
@@ -4590,11 +4594,20 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
output_file_map: dict[str, dict[str, Any]] | None = None,
) -> str | None:
ignored = {"stdout.log", "stderr.log", "hook.pid", "listener.pid"}
candidates = [
path
for path in output_file_paths
if Path(path).name not in ignored and Path(path).suffix.lower() not in (".pid", ".log", ".sh")
]
# Responses/wget manifests can contain thousands of paths. Parse each
# once instead of rebuilding Path objects for every preferred filename
# and extension group, each time a list row selects a preview.
candidates = []
names = {}
extensions = {}
for path in output_file_paths:
parsed_path = Path(path)
name, extension = parsed_path.name, parsed_path.suffix.lower()
if name in ignored or extension in (".pid", ".log", ".sh"):
continue
candidates.append(path)
names.setdefault(name.lower(), path)
extensions[path] = extension
if not candidates:
return None
@@ -4617,9 +4630,8 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
"article.json",
]
for preferred_name in preferred_names:
for candidate in candidates:
if Path(candidate).name.lower() == preferred_name:
return candidate
if preferred_name in names:
return names[preferred_name]
from archivebox.plugins.discovery import get_plugin_output_extension_preference
@@ -4630,7 +4642,7 @@ class ArchiveResult(ModelWithDeleteAfter, ModelWithOutputDir, ModelWithNotes):
(".mp4", ".webm", ".mp3", ".opus", ".ogg", ".wav"),
)
for ext_group in ext_groups:
group_candidates = [candidate for candidate in candidates if Path(candidate).suffix.lower() in ext_group]
group_candidates = [candidate for candidate in candidates if extensions[candidate] in ext_group]
if group_candidates:
return max(
group_candidates,
+4 -3
View File
@@ -74,8 +74,9 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
**snapshot_filter,
).update(retry_at=now, modified_at=now)
# Find rare uploads through hook_name's index before checking their parents;
# a correlated upload probe otherwise visits every sealed snapshot.
extension_upload_results = ArchiveResult.objects.filter(
snapshot_id=OuterRef("pk"),
hook_name=Snapshot.BROWSER_EXTENSION_UPLOAD_HOOK_NAME,
)
server_results = ArchiveResult.objects.filter(snapshot_id=OuterRef("pk")).exclude(
@@ -83,15 +84,15 @@ def recover_orchestrator_state(*, include_chrome: bool = False, crawl_id: str |
)
extension_only_snapshots = (
Snapshot.objects.filter(
id__in=extension_upload_results.values("snapshot_id"),
status=Snapshot.StatusChoices.SEALED,
crawl__status__in=[Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED],
**snapshot_filter,
)
.annotate(
has_extension_upload=Exists(extension_upload_results),
has_server_result=Exists(server_results),
)
.filter(has_extension_upload=True, has_server_result=False)
.filter(has_server_result=False)
)
# Older browser-extension uploads could win a race with runner startup:
# their successful external rows made the fresh Snapshot look finished
+62 -11
View File
@@ -1,3 +1,28 @@
"""Shared foreground signal and child-process cleanup contracts.
Shutdown intent and process exit are separate: takeover must preserve the
replacement owner's work, while a user abort must not later resume the crawl.
Callers own that policy; this module preserves the received signal and stops
only the process objects they explicitly supply.
Cooperative boundaries are essential, not just quieter exception handling.
A real Linux CI regression interrupted Popen.poll() with SIGINT and stranded
its wait lock. SIGKILL stopped the child (it became a zombie), but Popen.wait()
still timed out. Raising a different exception or waiting longer cannot repair
that interrupted critical section. Polling loops must record the signal first
and honor it between operations; async owners may cancel their own task while
their loop is running. Once exiting, the last-resort process cleanup must stay
synchronous: the original loop may be closed, and creating a replacement loop
cannot recover tasks or resources owned by it.
Keep immediate signal notices and repeated-signal force exit independent of
the event bus: it may already be draining or closed during shutdown. Resumable
pause/retry/skip belongs to the existing crawl controller and must not set
sticky command-shutdown state. See foreground_shutdown_signals below for both
cooperative and legacy blocking callers; do not change all callers' policy at
once without supplying their interruption boundaries.
"""
from __future__ import annotations
import os
@@ -23,7 +48,7 @@ _active_shutdown_state: ShutdownSignalState | None = None
def raise_if_shutdown_requested() -> None:
"""Let long foreground loops honor a signal even if Python ignored it once."""
"""Honor sticky shutdown intent at a caller-owned interruption boundary."""
if _active_shutdown_state and _active_shutdown_state.signal_name:
raise KeyboardInterrupt
@@ -48,14 +73,37 @@ def wait_popen_and_kill_children(
timeout: float,
kill_timeout: float = 2.0,
) -> None:
"""Wait for a Popen parent and then hard-kill any surviving descendants."""
"""Reap our own Popen child and stop its explicitly captured descendants.
A dead child still needs its parent to reap it. Keep wait failures visible;
bypassing Popen's lock would hide the interrupted-owner bug described above.
Descendants must still be cleaned up when that wait fails.
"""
try:
proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=kill_timeout)
kill_remaining_processes(children, timeout=kill_timeout)
try:
proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=kill_timeout)
except subprocess.TimeoutExpired as err:
try:
state = psutil.Process(proc.pid).as_dict(attrs=["pid", "ppid", "status", "num_threads", "cpu_times"])
except psutil.NoSuchProcess:
state = {"pid": proc.pid, "status": "exited"}
err.add_note(f"Process state after SIGKILL: {state}")
# A locked Popen wait can time out even after the OS process exits.
# Inspect CPython's lock only for diagnostics; never release it here.
waitpid_lock = getattr(proc, "_waitpid_lock", None)
err.add_note(
f"Popen state: returncode={proc.returncode}, "
f"waitpid_lock_locked={waitpid_lock.locked() if waitpid_lock is not None else None}, "
f"threads={[thread.name for thread in threading.enumerate()]}",
)
raise
finally:
# A parent that cannot be reaped must not leave its descendants running.
kill_remaining_processes(children, timeout=kill_timeout)
def wait_psutil_and_kill_children(
@@ -104,11 +152,14 @@ def foreground_shutdown_signals(
interrupt_handlers: dict[signal.Signals, Callable[[], None]] | None = None,
raise_on_first_signal: bool = True,
) -> Iterator[ShutdownSignalState]:
"""Install foreground signal handlers that print an immediate exit notice.
"""Keep shutdown intent visible until the owning foreground command unwinds.
Some log-tail loops intentionally swallow KeyboardInterrupt so that callers
can centralize cleanup in finally blocks. The handler writes the signal name
immediately, then raises KeyboardInterrupt to break out of the blocking read.
Signal state was added because inner loops can swallow KeyboardInterrupt
while their caller still needs to stop its children. Cooperative callers use
raise_on_first_signal=False and check raise_if_shutdown_requested() between
operations, or cancel their owned async task through on_signal. Raising from
the handler can otherwise interrupt library locks or partially sent I/O.
Blocking callers without a cooperative boundary retain immediate exceptions.
"""
global _active_shutdown_state
+3 -1
View File
@@ -775,7 +775,9 @@ def snapshot_index_row(context, link) -> str:
title = unescape(getattr(link, "title", "") or "")
title_text = title or url
tags_str = link.tags_str() if callable(getattr(link, "tags_str", None)) else getattr(link, "tags_str", "")
tag_html = "".join(f'<span class="snapshot-tag">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag)
tag_html = "".join(
f'<span class="snapshot-tag" title="{escape(tag)}">{escape(tag)}</span>' for tag in (tags_str or "").split(",") if tag
)
if tag_html:
tag_cell = f'<span class="snapshot-tags">{tag_html}</span>'
else:
+53 -20
View File
@@ -17,7 +17,6 @@ from django.contrib.auth import HASH_SESSION_KEY, SESSION_KEY, get_user_model
from django.contrib.auth.mixins import UserPassesTestMixin
from django.contrib.sessions.models import Session
from django.core import signing
from django.core.paginator import InvalidPage
from django.db.models import Case, IntegerField, Q, Value, When
from django.http import Http404, HttpRequest, HttpResponse, HttpResponseForbidden, QueryDict
from django.shortcuts import redirect, render
@@ -48,7 +47,6 @@ from archivebox.config.configset import BaseConfigSet
from archivebox.core.forms import AddLinkForm
from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
from archivebox.core.permissions import (
PERMISSIONS_PRIVATE,
PERMISSIONS_PUBLIC,
PERMISSIONS_UNLISTED,
can_view_snapshot,
@@ -92,6 +90,7 @@ from archivebox.plugins.discovery import (
from archivebox.plugins.forms import get_plugin_config_binary_urls
from archivebox.plugins.views import get_config_definition_link
from archivebox.progressmonitor.views import live_progress_view
from archivebox.progressmonitor.collection import collection_summary
from archivebox.search.config import (
get_search_mode,
get_search_mode_backend,
@@ -1238,14 +1237,23 @@ class PublicIndexView(ListView):
"status",
"output_size",
"permissions",
# Modern replay URLs need the filesystem version and owner's name.
# Deferring these caused four extra SELECTs per displayed snapshot.
"fs_version",
"crawl__created_by__username",
)
def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot] | None:
target_count = page_number * page_size
def _ordered_public_page_from_order_index(self, *, page_number: int, page_size: int) -> list[Snapshot]:
# One extra row determines Next without a full collection count. Cached
# totals can be stale while the runner adds/removes snapshots, so they
# must never decide whether the requested page exists.
target_count = page_number * page_size + 1
public_snapshots: list[Snapshot] = []
scanned = 0
chunk_size = max(self.public_page_scan_chunk_size, page_size)
ordered_snapshots = Snapshot.objects.order_by(*self.ordering).only(*self._base_public_snapshot_fields())
chunk_size = max(self.public_page_scan_chunk_size, page_size + 1)
ordered_snapshots = (
Snapshot.objects.select_related("crawl__created_by").order_by(*self.ordering).only(*self._base_public_snapshot_fields())
)
while len(public_snapshots) < target_count:
chunk = list(ordered_snapshots[scanned : scanned + chunk_size])
@@ -1261,20 +1269,31 @@ class PublicIndexView(ListView):
if self.request.GET.get("q", default="").strip():
return super().paginate_queryset(queryset, page_size)
public_count = self.get_exact_public_snapshot_count()
paginator = self.get_paginator(range(public_count), page_size)
page_kwarg = self.kwargs.get(self.page_kwarg)
page_query = self.request.GET.get(self.page_kwarg)
page_number = page_kwarg or page_query or 1
try:
page = paginator.page(page_number)
except InvalidPage as err:
page_number = int(page_number)
if page_number < 1:
raise ValueError("Page must be positive")
except (ValueError, TypeError) as err:
raise Http404(f"Invalid page ({page_number}): {err}") from err
object_list = self._ordered_public_page_from_order_index(page_number=page.number, page_size=page_size)
page.object_list = object_list
return paginator, page, object_list, page.has_other_pages()
snapshots = self._ordered_public_page_from_order_index(page_number=page_number, page_size=page_size)
if not snapshots and page_number > 1:
raise Http404(f"Invalid page ({page_number}): No snapshots")
observed_count = (page_number - 1) * page_size + len(snapshots)
self.public_snapshot_count = observed_count
if len(snapshots) > page_size:
summary = collection_summary(self.request.user, public=True)
cached_count = summary["snapshots"] if summary is not None else None
self.public_snapshot_count = cached_count if cached_count is not None and cached_count >= observed_count else None
paginator = self.get_paginator(range(observed_count), page_size)
page = paginator.page(page_number)
page.object_list = snapshots[:page_size]
return paginator, page, page.object_list, page.has_other_pages()
def get_context_data(self, **kwargs):
runtime_config = self.__dict__.get("runtime_config")
@@ -1303,10 +1322,16 @@ class PublicIndexView(ListView):
and search_mode_backend
and context["paginator"].count == 0,
)
context["public_snapshot_count"] = context["paginator"].count if query else self.public_snapshot_count
count = context["public_snapshot_count"]
context["public_page_count"] = (
max(1, (count + context["paginator"].per_page - 1) // context["paginator"].per_page) if count is not None else None
)
snapshots = list(context.get("object_list") or ())
from archivebox.plugins.output_groups import plugin_output_sizes
all_results_by_snapshot = {str(snapshot.id): [] for snapshot in snapshots}
snapshots_by_id = {str(snapshot.id): snapshot for snapshot in snapshots}
icons_by_snapshot: dict[str, dict[str, ArchiveResult]] = {str(snapshot.id): {} for snapshot in snapshots}
tag_names_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
list_icon_paths_by_snapshot: dict[str, list[str]] = {str(snapshot.id): [] for snapshot in snapshots}
@@ -1343,8 +1368,17 @@ class PublicIndexView(ListView):
"output_size",
)
)
for result in results.iterator(chunk_size=1000):
# This is already bounded by the displayed snapshot page. Finish
# the SELECT before parsing output manifests: a streaming cursor
# otherwise keeps SQLite's read lock throughout rendering.
for result in list(results):
snapshot_key = str(result.snapshot_id)
# Thumbnail URLs traverse result.snapshot too; share the page's
# already-loaded parent rather than fetching it once per result.
result.snapshot = snapshots_by_id[snapshot_key]
# These read-only page objects are shared by icons, sizes, and
# thumbnails. Normalize each manifest once for this render.
result._render_output_file_map = result.output_file_map()
all_results_by_snapshot[snapshot_key].append(result)
progress = progress_by_snapshot[snapshot_key]
progress["total"] += 1
@@ -1376,13 +1410,12 @@ class PublicIndexView(ListView):
context["object_list"] = snapshots
return context
def get_exact_public_snapshot_count(self) -> int:
hidden_count = Snapshot.objects.filter(permissions=PERMISSIONS_PRIVATE).count()
hidden_count += Snapshot.objects.filter(permissions=PERMISSIONS_UNLISTED).count()
return Snapshot.objects.count() - hidden_count
def get_queryset(self, **kwargs):
qs = public_snapshots_queryset(super().get_queryset(**kwargs)).only(*self._base_public_snapshot_fields())
qs = (
public_snapshots_queryset(super().get_queryset(**kwargs))
.select_related("crawl__created_by")
.only(*self._base_public_snapshot_fields())
)
query = self.request.GET.get("q", default="").strip()
if not query:
+1 -1
View File
@@ -717,7 +717,7 @@ class InlineTagEditorWidget(TagEditorWidget):
)
pills_html += f'''
<span class="tag-pill" data-tag="{self._escape(td["name"])}" data-tag-id="{td["id"]}" style="{self._tag_style(td["name"])}">
<a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link">{self._escape(td["name"])}</a>
<a href="/admin/core/snapshot/?tags__id__exact={td["id"]}" class="tag-link" title="{self._escape(td["name"])}">{self._escape(td["name"])}</a>
{remove_button}
</span>
'''
+22 -41
View File
@@ -221,35 +221,21 @@ class Machine(ModelWithHealthStats):
app_label = "machine"
@classmethod
def current(cls, refresh: bool = False) -> Machine:
global _CURRENT_MACHINE
if refresh:
_CURRENT_MACHINE = None
if _CURRENT_MACHINE:
if timezone.now() < _CURRENT_MACHINE.modified_at + timedelta(seconds=MACHINE_RECHECK_INTERVAL):
# Reconciliation can save() and clear the shared cache; keep
# this call's instance so we never return None after that save.
machine = _CURRENT_MACHINE
# One-time-per-process reconciliation between ArchiveBox.conf
# and Machine.config. Fast-path: bool check + early-return when
# the sync has already run, so the cached-machine return path
# stays sub-microsecond.
try:
from archivebox.config.collection import sync_machine_and_file
def current_readonly(cls) -> Machine | None:
"""Read the current machine without registration, refresh, or config writes."""
if _CURRENT_MACHINE is not None:
return _CURRENT_MACHINE
return cls.objects.filter(guid=get_host_guid()).first()
sync_machine_and_file(machine)
except Exception:
pass
return machine
else:
_CURRENT_MACHINE = None
@classmethod
def current(cls) -> Machine:
global _CURRENT_MACHINE
if _CURRENT_MACHINE is not None:
return _CURRENT_MACHINE
host_guid = get_host_guid()
# save() intentionally invalidates _CURRENT_MACHINE so config edits
# made through other instances become visible in this process. Host
# refresh (after seven days), sanitation, and reconciliation can all
# save here, so the working instance must not live only in that cache.
# Otherwise a routine refresh leaves startup dereferencing None.
# Host detection, sanitation, and config reconciliation run only on
# initial registration, never on cached reads.
try:
machine = cls.objects.get(guid=host_guid)
except cls.DoesNotExist:
@@ -296,17 +282,14 @@ class Machine(ModelWithHealthStats):
],
)
machine = cls._sanitize_config(machine)
# Same one-time sync as the cached-return path. Triggers here on the
# very first ``Machine.current()`` call in a process before any
# cached return can occur.
# Reconcile once, on the first Machine.current() call in this process.
try:
from archivebox.config.collection import sync_machine_and_file
sync_machine_and_file(machine)
except Exception:
pass
# Publish only after all save-capable steps; publishing earlier would
# let those steps invalidate the cache we are in the middle of filling.
# Publish only after initialization is complete.
_CURRENT_MACHINE = machine
return machine
@@ -377,18 +360,15 @@ class Machine(ModelWithHealthStats):
if update_fields is not None:
kwargs["update_fields"] = tuple(dict.fromkeys([*update_fields, "config"]))
# Drop the ``Machine.current()`` module-level cache on every save so
# config edits (admin form, from_json, etc.) become live in the same
# process without waiting out the 7-day ``MACHINE_RECHECK_INTERVAL``.
# The cache reads ``_CURRENT_MACHINE.modified_at`` and that value
# never moves forward on the cached object even when the row is
# updated in the DB, so without this we'd keep serving stale
# ``machine.config`` (incl. ``BASE_URL``) until the worker restarts.
# Apply explicit edits to the cached instance without making the next
# read repeat host detection, sanitation, or config reconciliation.
update_fields = kwargs.get("update_fields")
super().save(*args, **kwargs)
global _CURRENT_MACHINE
if _CURRENT_MACHINE is not None and _CURRENT_MACHINE.pk == self.pk:
_CURRENT_MACHINE = None
for field in self._meta.concrete_fields:
if field.attname in self.__dict__ and (update_fields is None or field.name in update_fields):
setattr(_CURRENT_MACHINE, field.attname, self.__dict__[field.attname])
# Mirror Machine.config into ArchiveBox.conf so the two stores stay
# 1:1. Skipped when ``update_fields`` is set and doesn't touch
@@ -443,7 +423,7 @@ class NetworkInterface(ModelWithHealthStats):
@classmethod
def current(cls, refresh: bool = False) -> NetworkInterface:
global _CURRENT_INTERFACE
machine = Machine.current(refresh=refresh)
machine = Machine.current()
if _CURRENT_INTERFACE and _CURRENT_INTERFACE.machine_id == machine.id:
if not refresh:
# Callers that pass refresh=False are asking for attribution to
@@ -1778,7 +1758,8 @@ class Process(ModelWithDeleteAfter, models.Model):
return None
# Can't validate processes on other machines
if self.machine_id != Machine.current().id:
machine = Machine.current_readonly()
if machine is None or self.machine_id != machine.id:
return None
try:
+2
View File
@@ -526,6 +526,8 @@ HISTORICAL_GHOST_MIGRATIONS: frozenset[tuple[str, str]] = frozenset(
("core", "0053_remove_snapshottag_snapshot_old"),
("core", "0054_alter_snapshot_timestamp"),
("core", "0055_alter_tag_slug"),
# Dev-only progress indexes removed in favor of existing-index queries.
("core", "0056_progress_covering_indexes"),
("core", "0056_remove_tag_uuid"),
("core", "0057_rename_id_tag_old_id"),
("core", "0058_alter_tag_old_id"),
+72 -8
View File
@@ -12,9 +12,62 @@ from django.db.models import Count, Sum
_refresh_lock = Lock()
def collection_summary(user):
def crawl_summary(crawl_id, *, snapshot_id=None):
"""Sample sealed snapshot metadata without adding table scans to polls."""
key = f"progress-crawl:{crawl_id}:{snapshot_id or 'all'}"
summary = cache.get(key)
if (summary is None or time() - summary["sampled_at"] >= 30) and _refresh_lock.acquire(blocking=False):
if not cache.add(f"{key}:refresh", True, timeout=30):
_refresh_lock.release()
return summary
def refresh():
try:
close_old_connections()
from archivebox.core.models import Snapshot
snapshots = Snapshot.objects.filter(crawl_id=crawl_id, status=Snapshot.StatusChoices.SEALED)
if snapshot_id is not None:
snapshots = snapshots.filter(pk=snapshot_id)
snapshots = snapshots.order_by("modified_at").values_list("modified_at", "downloaded_at", "output_size")
totals = {"snapshots": 0, "cancelled": 0, "bytes": 0}
modified_at, offset = None, 0
while True:
batch_scope = snapshots.filter(modified_at__gte=modified_at) if modified_at is not None else snapshots
# Use (crawl, status, modified_at), releasing the SQLite read
# cursor after each small batch. Never hold a cursor over the
# whole crawl, even in this background worker.
batch = list(batch_scope[offset : offset + 100])
for _, downloaded_at, size in batch:
totals["snapshots"] += 1
totals["cancelled"] += downloaded_at is None
totals["bytes"] += size or 0
if len(batch) < 100:
break
last_modified_at = batch[-1][0]
# Only offset within ties; otherwise advance the indexed
# timestamp range instead of rescanning previous pages.
offset = (offset if last_modified_at == modified_at else 0) + sum(row[0] == last_modified_at for row in batch)
modified_at = last_modified_at
totals["sampled_at"] = time()
cache.set(key, totals, timeout=3600)
except Exception:
logging.getLogger(__name__).exception("Could not refresh crawl totals")
finally:
connections.close_all()
_refresh_lock.release()
try:
Thread(target=refresh, name="crawl-summary", daemon=True).start()
except RuntimeError:
_refresh_lock.release()
logging.getLogger(__name__).exception("Could not start crawl totals refresh")
return summary
def collection_summary(user, *, public=False):
user_id = None if user.is_superuser else user.pk
key = f"progress-collection:{'all' if user_id is None else user_id}"
key = "progress-collection:public" if public else f"progress-collection:{'all' if user_id is None else user_id}"
summary = cache.get(key)
# Even an indexed SUM scans the index. Never make an HTTP request wait for
# that scan on a large collection. Serve stale totals while one worker refreshes;
@@ -32,12 +85,23 @@ def collection_summary(user):
from archivebox.core.models import Snapshot
snapshots = Snapshot.objects.all()
if user_id is not None:
snapshots = snapshots.filter(crawl__created_by_id=user_id)
# COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
# Snapshot.output_size is maintained when ArchiveResult sizes change.
totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
totals["bytes"] = totals["bytes"] or 0
if public:
from archivebox.core.permissions import PERMISSIONS_PUBLIC
# SQLite can choose a table scan for equality after ANALYZE
# on an all-public archive. Equal range bounds keep the count
# on the existing permissions index. It is still O(n), so
# public pages share this background cache, never await it.
totals = snapshots.filter(permissions__gte=PERMISSIONS_PUBLIC, permissions__lte=PERMISSIONS_PUBLIC).aggregate(
snapshots=Count("permissions"),
)
else:
if user_id is not None:
snapshots = snapshots.filter(crawl__created_by_id=user_id)
# COUNT(*) + SUM(output_size) can use Snapshot's covering size index.
# Snapshot.output_size is maintained when ArchiveResult sizes change.
totals = snapshots.aggregate(snapshots=Count("*"), bytes=Sum("output_size"))
totals["bytes"] = totals.get("bytes") or 0
totals["sampled_at"] = time()
cache.set(key, totals, timeout=3600)
except Exception:
+72 -90
View File
@@ -7,12 +7,11 @@ from typing import Literal
from abx_dl.events import PROCESS_EXIT_SKIPPED
from django.conf import settings
from django.db import DatabaseError
from django.db.models import CharField, Count, Q, Sum
from django.db.models import CharField, Count, Q
from django.db.models.functions import Cast
from django.http import HttpResponse, JsonResponse
from django.utils import timezone
from archivebox.config.common import get_config
from archivebox.core.permissions import can_view_snapshot, is_admin_user
from archivebox.core.routes_util import build_snapshot_detail_url, build_snapshot_url, get_api_base_url
from archivebox.misc.logging_util import printable_filesize
@@ -182,53 +181,37 @@ def live_progress_view(request, *, authorized_snapshot=None):
running_screencasts = {}
def screencast_frame_url(crawl_id: str, crawl_dir: Path) -> str:
frame_path = crawl_dir / "chrome_screencast" / "latest.jpg"
try:
frame_stat = frame_path.stat()
except OSError:
return ""
if frame_stat.st_size <= 0:
return ""
def screencast_frame_url(crawl_id: str) -> str:
started_at = running_screencasts.get(crawl_id)
if started_at is None or frame_stat.st_mtime < started_at.timestamp():
if started_at is None:
return ""
rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={frame_stat.st_mtime_ns}"
# File existence/freshness belongs to the image request, never to
# the one-second JSON poll (archive storage may be remote).
rel = f"/api/v1/crawls/crawl/{crawl_id}/files/chrome_screencast/latest.jpg?v={int(now.timestamp())}&after={started_at.timestamp()}"
return f"{api_base}{rel}" if api_base else rel
machine_id = Machine.current().id
machine = Machine.current_readonly()
machine_id = machine.id if machine is not None else None
orchestrator_proc = (
Process.objects.filter(
machine_id=machine_id,
process_type=Process.TypeChoices.ORCHESTRATOR,
status=Process.StatusChoices.RUNNING,
)
.only("id", "pid", "started_at", "machine_id", "process_type", "status")
.only("id", "pid", "started_at", "machine_id", "process_type", "status", "env", "cmd")
.order_by("-started_at")
.first()
if machine_id is not None
else None
)
runner_worker = None
orchestrator_proc_running = bool(orchestrator_proc and orchestrator_proc.is_running)
if not orchestrator_proc_running:
try:
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker
supervisor = get_existing_supervisord_process(quiet=True)
runner_worker = get_worker(supervisor, "worker_runner") if supervisor else None
except (OSError, RuntimeError, TimeoutError):
runner_worker = None
runner_worker_running = bool(runner_worker and runner_worker.get("statename") in ("STARTING", "RUNNING"))
runner_worker_pid = runner_worker.get("pid") if runner_worker else None
orchestrator_running = orchestrator_proc_running or runner_worker_running
orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else runner_worker_pid
orchestrator_running = orchestrator_proc_running
orchestrator_pid = orchestrator_proc.pid if orchestrator_proc_running and orchestrator_proc else None
# Get model counts by status
crawl_status_counts = Crawl.status_counts(
crawl_scope,
(Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED, Crawl.StatusChoices.PAUSED),
(Crawl.StatusChoices.QUEUED, Crawl.StatusChoices.STARTED),
)
crawls_queued = crawl_status_counts.get(Crawl.StatusChoices.QUEUED, 0)
crawls_active = crawl_status_counts.get(Crawl.StatusChoices.STARTED, 0)
@@ -242,7 +225,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
snapshot_status_counts = Snapshot.status_counts(
snapshot_scope,
Snapshot.OPEN_STATES,
Snapshot.RUNNABLE_STATES,
)
snapshots_queued = snapshot_status_counts.get(Snapshot.StatusChoices.QUEUED, 0)
snapshots_active = snapshot_status_counts.get(Snapshot.StatusChoices.STARTED, 0)
@@ -252,15 +235,24 @@ def live_progress_view(request, *, authorized_snapshot=None):
ArchiveResult.StatusChoices.QUEUED,
ArchiveResult.StatusChoices.STARTED,
)
archiveresult_status_counts = ArchiveResult.status_counts(archiveresult_scope, result_statuses)
# Start from open results using the existing status index. Filtering by
# plugin first scans historical runs, including all completed results.
result_counts = list(
archiveresult_scope.filter(status__in=result_statuses).order_by().values("status", "plugin").annotate(count=Count("*")),
)
archiveresult_status_counts = {
status: sum(row["count"] for row in result_counts if row["status"] == status) for status in result_statuses
}
download_scope = archiveresult_scope.filter(
plugin__in=download_plugin_names,
snapshot__status__in=Snapshot.RUNNABLE_STATES,
snapshot__crawl__status__in=Crawl.RUNNABLE_STATES,
)
indexing_scope = archiveresult_scope.filter(plugin__in=indexing_plugin_names)
download_status_counts = ArchiveResult.status_counts(download_scope, result_statuses)
indexing_status_counts = ArchiveResult.status_counts(indexing_scope, result_statuses)
indexing_status_counts = {
status: sum(row["count"] for row in result_counts if row["status"] == status and row["plugin"] in indexing_plugin_names)
for status in result_statuses
}
archiveresults_queued = archiveresult_status_counts.get(ArchiveResult.StatusChoices.QUEUED, 0)
archiveresults_active = archiveresult_status_counts.get(ArchiveResult.StatusChoices.STARTED, 0)
@@ -296,21 +288,27 @@ def live_progress_view(request, *, authorized_snapshot=None):
.values(*active_crawl_fields)
.order_by("-modified_at")[:max_active_crawls],
)
paused_crawls = list(
crawl_scope.filter(
Q(status=Crawl.StatusChoices.PAUSED, created_at__gte=paused_crawl_cutoff)
| Q(
status=Crawl.StatusChoices.PAUSED,
snapshot_set__status__in=Snapshot.RUNNABLE_STATES,
snapshot_set__retry_at__lte=now,
)
| Q(
status=Crawl.StatusChoices.PAUSED,
snapshot_set__archiveresult__status=ArchiveResult.StatusChoices.QUEUED,
),
# Start with runnable children, not every historical snapshot belonging
# to a paused crawl. These uncorrelated subqueries run once per poll.
due_paused_snapshots = (
Snapshot.objects.filter(
status__in=Snapshot.RUNNABLE_STATES,
retry_at__lte=now,
)
.order_by()
.values("crawl_id")
)
queued_paused_results = (
ArchiveResult.objects.filter(
status=ArchiveResult.StatusChoices.QUEUED,
)
.order_by()
.values("snapshot__crawl_id")
)
paused_crawls = list(
crawl_scope.filter(status=Crawl.StatusChoices.PAUSED)
.filter(Q(created_at__gte=paused_crawl_cutoff) | Q(id__in=due_paused_snapshots) | Q(id__in=queued_paused_results))
.values(*active_crawl_fields)
.distinct()
.order_by("-modified_at")[:max_active_crawls],
)
queued_crawls = list(
@@ -324,52 +322,29 @@ def live_progress_view(request, *, authorized_snapshot=None):
crawl["persona_id"] = str(crawl["persona_id"])
persona_details_by_id: dict[str, dict[str, str]] = {}
persona_details_by_name: dict[str, dict[str, str]] = {}
persona_objects_by_id = {}
persona_objects_by_name = {}
persona_ids = {crawl["persona_id"] for crawl in active_crawls_list if crawl["persona_id"]}
persona_names = {"Default"} if any(not crawl["persona_id"] for crawl in active_crawls_list) else set()
if persona_ids or persona_names:
from archivebox.personas.models import Persona
for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name", "config"):
for persona in Persona.objects.filter(Q(id__in=persona_ids) | Q(name__in=persona_names)).only("id", "name"):
persona_details = {
"name": persona.name,
"admin_url": f"/admin/personas/persona/{persona.pk}/change/",
}
persona_details_by_id[str(persona.id)] = persona_details
persona_details_by_name[persona.name] = persona_details
persona_objects_by_id[str(persona.id)] = persona
persona_objects_by_name[persona.name] = persona
active_crawl_ids = [crawl["id"] for crawl in active_crawls_list]
active_crawl_objects = {}
if active_crawl_ids:
for crawl_obj in Crawl.objects.filter(id__in=active_crawl_ids).select_related("created_by", "persona"):
crawl_obj._runtime_config = request_config
active_crawl_objects[str(crawl_obj.id)] = crawl_obj
snapshot_counts_by_crawl: dict[str, dict[str, int]] = {str(crawl_id): {} for crawl_id in active_crawl_ids}
cancelled_snapshot_counts_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
crawl_output_sizes_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
queued_snapshot_overflow_by_crawl: dict[str, int] = {str(crawl_id): 0 for crawl_id in active_crawl_ids}
active_snapshot_scope = snapshot_scope.filter(crawl_id__in=active_crawl_ids)
if active_crawl_ids:
for row in active_snapshot_scope.values("crawl_id", "status").annotate(count=Count("id")):
snapshot_counts_by_crawl.setdefault(str(row["crawl_id"]), {})[row["status"]] = row["count"]
for row in (
active_snapshot_scope.filter(status=Snapshot.StatusChoices.SEALED, downloaded_at__isnull=True)
.values("crawl_id")
.annotate(count=Count("id"))
):
cancelled_snapshot_counts_by_crawl[str(row["crawl_id"])] = row["count"]
for row in (
active_snapshot_scope.filter(
status=Snapshot.StatusChoices.SEALED,
)
.values("crawl_id")
.annotate(size=Sum("output_size"))
):
crawl_output_sizes_by_crawl[str(row["crawl_id"])] = int(row["size"] or 0)
# The existing (crawl, status, modified_at) index covers these counts.
# Byte totals and cancellation details require table reads, so sample
# those separately and share cached values between viewer tabs.
for row in active_snapshot_scope.order_by().values("crawl_id", "status").annotate(count=Count("*")):
crawl_id = str(row["crawl_id"])
snapshot_counts_by_crawl[crawl_id][row["status"]] = row["count"]
crawl_process_pids: dict[str, int] = {}
snapshot_process_pids: dict[str, int] = {}
@@ -588,7 +563,14 @@ def live_progress_view(request, *, authorized_snapshot=None):
completed_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.SEALED, 0)
started_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.STARTED, 0)
pending_snapshots = crawl_snapshot_counts.get(Snapshot.StatusChoices.QUEUED, 0)
cancelled_snapshots = cancelled_snapshot_counts_by_crawl.get(crawl_id, 0)
from archivebox.progressmonitor.collection import crawl_summary
totals = (
crawl_summary(crawl_id, snapshot_id=scoped_snapshot.pk if scoped_snapshot is not None else None)
if completed_snapshots
else {"snapshots": 0, "cancelled": 0, "bytes": 0}
)
cancelled_snapshots = totals["cancelled"] if totals is not None else None
# Count URLs in the crawl (for when snapshots haven't been created yet)
urls_count = 0
@@ -607,7 +589,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
crawl_setup_completed = sum(1 for item in crawl_setup_plugins if item.get("status") == "succeeded")
crawl_setup_failed = sum(1 for item in crawl_setup_plugins if item.get("status") == "failed")
crawl_setup_pending = sum(1 for item in crawl_setup_plugins if item.get("status") == "queued")
crawl_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
crawl_screencast_url = screencast_frame_url(crawl_id)
crawl_screencast_link = f"/admin/crawls/crawl/{crawl_id.replace('-', '')}/change/" if crawl_screencast_url else ""
# Get active snapshots for this crawl (already prefetched)
@@ -661,7 +643,7 @@ def live_progress_view(request, *, authorized_snapshot=None):
snapshot_preview_url = snapshot_favicon_url
if snapshot["status"] == Snapshot.StatusChoices.STARTED:
snapshot_screencast_url = screencast_frame_url(crawl_id, active_crawl_objects[crawl_id].output_dir)
snapshot_screencast_url = screencast_frame_url(crawl_id)
snapshot_screencast_link = snapshot_view_url(snapshot) if snapshot_screencast_url else ""
def plugin_sort_key(ar):
@@ -809,18 +791,18 @@ def live_progress_view(request, *, authorized_snapshot=None):
persona_details = persona_details_by_id.get(str(crawl["persona_id"])) if crawl["persona_id"] else None
persona_name = persona_details["name"] if persona_details else "Default"
persona_details = persona_details or persona_details_by_name.get(persona_name)
crawl_output_size = crawl_output_sizes_by_crawl.get(crawl_id, 0)
avg_snapshot_size = int(crawl_output_size / completed_snapshots) if completed_snapshots else 0
crawl_obj = active_crawl_objects[crawl_id]
effective_crawl_config = get_config(crawl=crawl_obj, resolve_plugins=False)
max_urls = int(effective_crawl_config.CRAWL_MAX_URLS or 0)
crawl_max_size = int(effective_crawl_config.CRAWL_MAX_SIZE or 0)
crawl_timeout = int(effective_crawl_config.CRAWL_TIMEOUT or 0)
snapshot_max_size = int(effective_crawl_config.SNAPSHOT_MAX_SIZE or 0)
crawl_output_size = totals["bytes"] if totals is not None else None
avg_snapshot_size = int(totals["bytes"] / max(totals["snapshots"], 1)) if totals is not None else None
# These crawl-scope limits are frozen in Crawl.config. Rendering
# them must not resolve personas, probe binaries, or reread config.
max_urls = int(crawl["config"].get("CRAWL_MAX_URLS", request_config.CRAWL_MAX_URLS) or 0)
crawl_max_size = int(crawl["config"].get("CRAWL_MAX_SIZE", request_config.CRAWL_MAX_SIZE) or 0)
crawl_timeout = int(crawl["config"].get("CRAWL_TIMEOUT", request_config.CRAWL_TIMEOUT) or 0)
snapshot_max_size = int(crawl["config"].get("SNAPSHOT_MAX_SIZE", request_config.SNAPSHOT_MAX_SIZE) or 0)
# Check if retry_at is in the future (would prevent worker from claiming)
retry_at_future = crawl["retry_at"] > now if crawl["retry_at"] else False
is_paused = crawl_obj.is_paused
is_paused = crawl["status"] == Crawl.StatusChoices.PAUSED
seconds_until_retry = (
0 if is_paused else int((crawl["retry_at"] - now).total_seconds()) if crawl["retry_at"] and retry_at_future else 0
)
@@ -860,8 +842,8 @@ def live_progress_view(request, *, authorized_snapshot=None):
"max_snapshot_size_display": printable_filesize(snapshot_max_size) if snapshot_max_size else "unlimited",
"crawl_output_size": crawl_output_size,
"avg_snapshot_size": avg_snapshot_size,
"crawl_output_size_display": printable_filesize(crawl_output_size) if crawl_output_size else "0 B",
"avg_snapshot_size_display": printable_filesize(avg_snapshot_size) if avg_snapshot_size else "0 B",
"crawl_output_size_display": printable_filesize(crawl_output_size or 0) if totals is not None else "…",
"avg_snapshot_size_display": printable_filesize(avg_snapshot_size or 0) if totals is not None else "…",
"tags": crawl_tags,
"urls_count": urls_count,
"total_snapshots": total_snapshots,
+3 -3
View File
@@ -130,8 +130,8 @@ class ResourceAdmission:
slots = min(slots, unpromised // observed_cost)
return max(1, slots) if active == 0 and configured else slots
def memory_headroom(self) -> tuple[int, int, int] | None:
"""Return workload RAM+swap usage, effective headroom, and host headroom."""
def memory_headroom(self) -> tuple[int, int] | None:
"""Return workload RAM+swap usage and effective available headroom."""
try:
meminfo = {
name: int(value.split()[0]) * 1024
@@ -173,7 +173,7 @@ class ResourceAdmission:
used_bytes += child.memory_info().rss
except (psutil.Error, OSError):
continue
return used_bytes, available, host_available
return used_bytes, available
def observe_memory_stalls(self, source: str, total: int, avg10: float, sampled_at: float) -> bool:
previous = self._memory_stalls.get(source)
+19 -10
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import contextvars
import json
import logging
import os
import signal
import shutil
@@ -233,7 +234,7 @@ class CrawlRunner:
self.snapshot_semaphore = asyncio.Semaphore(1)
self.max_concurrent_snapshots = 1
self._warmup_snapshot_id: str | None = None
self._memory_baseline: tuple[int, int] | None = None
self._memory_baseline: int | None = None
self._memory_peak_cost = 0
self._observed_snapshot_cost: int | None = None
self._resource_deferred = False
@@ -506,7 +507,7 @@ class CrawlRunner:
return
if self._observed_snapshot_cost is None and self._warmup_snapshot_id is None:
observation = resource_admission.memory_headroom()
self._memory_baseline = (observation[0], observation[2]) if observation is not None else None
self._memory_baseline = observation[0] if observation is not None else None
self._memory_peak_cost = 0
self._warmup_snapshot_id = snapshot_id
if isinstance(current_event, CrawlStartEvent):
@@ -596,15 +597,13 @@ class CrawlRunner:
return
observation = resource_admission.memory_headroom()
if observation is not None:
used, _available, host_available = observation
baseline_used, baseline_host_available = self._memory_baseline
# Host headroom sees helpers outside ArchiveBox's cgroup. Keep a
# high-water cost as later URLs may use more than the first one.
# Never divide concurrent growth by task count: their pages vary.
used, _available = observation
# Only attribute this workload's growth to a capture. Other jobs on
# the host reduce available headroom, but are not a per-capture cost
# to multiply again for every snapshot we want to admit.
self._memory_peak_cost = max(
self._memory_peak_cost,
used - baseline_used,
baseline_host_available - host_available,
used - self._memory_baseline,
)
if self._observed_snapshot_cost is not None:
self._observed_snapshot_cost = max(self._observed_snapshot_cost, self._memory_peak_cost)
@@ -615,6 +614,13 @@ class CrawlRunner:
self._observe_snapshot_memory()
if self._memory_peak_cost > 0:
self._observed_snapshot_cost = self._memory_peak_cost
headroom = resource_admission.memory_headroom()
logging.getLogger(__name__).info(
"Snapshot admission: concurrency limit=%s, observed memory growth=%s bytes, available=%s bytes",
self.max_concurrent_snapshots,
self._observed_snapshot_cost,
headroom[1] if headroom else None,
)
else:
# No observed cost cannot justify parallel admission. Probe the
# next real snapshot while the queue keeps moving one at a time.
@@ -716,6 +722,9 @@ class CrawlRunner:
from archivebox.machine.models import Machine, NetworkInterface, Process
self.primary_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ""
# Validate saved binary paths at the execution boundary, including
# config edited since this process first registered its Machine.
machine = Machine._sanitize_config(Machine.current())
current_iface = NetworkInterface.current(refresh=not self.allow_maintenance_on_inactive_crawl)
current_process = Process.current()
if current_process.iface_id != current_iface.id or current_process.machine_id != current_iface.machine_id:
@@ -724,7 +733,7 @@ class CrawlRunner:
current_process.save(update_fields=["iface", "machine", "modified_at"])
self.persona = self.crawl.resolve_persona()
self.base_config = get_config(crawl=self.crawl, overrides=self.config_overrides)
self.derived_config = dict(Machine.current().config or {})
self.derived_config = dict(machine.config or {})
self.crawl_output_dir = str(self.crawl.output_dir)
if self.persona:
self.base_config.update(
+15
View File
@@ -1527,6 +1527,9 @@
display: inline-flex;
flex-wrap: wrap;
align-items: center;
min-width: 0;
max-width: 100%;
box-sizing: border-box;
gap: 4px;
padding: 2px 4px;
background: transparent;
@@ -1540,15 +1543,27 @@
flex-wrap: wrap;
gap: 4px;
align-items: center;
min-width: 0;
max-width: 100%;
}
.tag-editor-inline .tag-pill {
min-width: 0;
max-width: min(160px, 100%);
box-sizing: border-box;
padding: 2px 6px 2px 8px;
font-size: 11px;
border-radius: 12px;
}
.tag-editor-inline .tag-link {
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
}
.tag-editor-inline .tag-remove-btn {
flex-shrink: 0;
width: 14px;
height: 14px;
font-size: 12px;
@@ -79,6 +79,13 @@
}
.cards .card .card-tags span {
display: inline-block;
max-width: min(160px, 100%);
box-sizing: border-box;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
vertical-align: middle;
pointer-events: auto;
padding: 2px 5px;
border-radius: 5px;
opacity: 0.95;
@@ -283,16 +290,16 @@
<div class="card-media">
<a href="{% snapshot_detail_url obj %}" class="card-thumbnail">
{% snapshot_thumbnail obj admin=True width="100%" height="245px" %}
</a>
{% if obj.tags_str %}
<div class="card-tags">
{% for tag in obj.tags_str|split:',' %}
<span>
<span title="{{tag}}">
{{tag}}
</span>
{% endfor %}
</div>
{% endif %}
</a>
</div>
<div class="card-outputs">
{% if obj.status == 'started' %}
+22 -12
View File
@@ -198,21 +198,29 @@
}
.snapshot-preview-cell {
position: relative;
width: 110px;
min-width: 116px;
vertical-align: middle;
}
.snapshot-preview-cell a {
position: absolute;
top: 10px;
bottom: 10px;
display: inline-flex;
align-items: center;
justify-content: center;
width: 102px;
height: 102px;
line-height: 0;
vertical-align: middle;
}
.snapshot-preview-cell .snapshot-thumbnail {
width: 100% !important;
height: 100% !important;
}
#content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
#content .public-snapshot-list .snapshot-preview-cell img.snapshot-preview.screenshot {
box-sizing: border-box;
@@ -330,9 +338,10 @@
}
.snapshot-tag {
display: inline-flex;
align-items: center;
max-width: 160px;
display: block;
min-width: 0;
max-width: min(160px, 100%);
box-sizing: border-box;
padding: 2px 7px;
border: 1px solid #bfdbfe;
border-radius: 999px;
@@ -585,7 +594,8 @@
.snapshot-preview-cell a {
width: 82px;
height: 82px;
top: 7px;
bottom: 7px;
}
#content .public-snapshot-list .snapshot-preview-cell .snapshot-preview,
@@ -648,8 +658,8 @@
{% if public_search_stream_pending %}
<span class="archivebox-search-stream-status" role="status" aria-live="polite">Searching...</span>
{% else %}
{{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }}
<span>(page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }})</span>
{{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }}{% endif %}
<span>(page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %})</span>
{% endif %}
</div>
</div>
@@ -659,7 +669,7 @@
<tr>
<th class="snapshot-time">Saved</th>
<th class="snapshot-preview-cell">Preview</th>
<th>Snapshot ({{ page_obj.paginator.count }})</th>
<th>Snapshot{% if public_snapshot_count != None %} ({{ public_snapshot_count }}){% endif %}</th>
<th class="snapshot-tags-cell">Tags</th>
<th class="snapshot-status-cell">Status</th>
<th class="snapshot-files-cell">Files</th>
@@ -687,7 +697,7 @@
</table>
</div>
<div class="public-pagination">
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }} of {{ page_obj.paginator.count }} total
Showing {{ page_obj.start_index }}-{{ page_obj.end_index }}{% if public_snapshot_count != None %} of {{ public_snapshot_count }} total{% endif %}
<br>
<span class="step-links">
{% if page_obj.has_previous %}
@@ -696,14 +706,14 @@
{% endif %}
<span class="current">
Page {{ page_obj.number }} of {{ page_obj.paginator.num_pages }}
Page {{ page_obj.number }}{% if public_page_count != None %} of {{ public_page_count }}{% endif %}
</span>
{% if page_obj.has_next %}
<a href="?{% url_replace page=page_obj.next_page_number %}">next</a>
{% endif %}
{% if page_obj.has_next %}
<a href="?{% url_replace page=page_obj.paginator.num_pages %}">last &raquo;</a>
{% if page_obj.has_next and public_page_count != None %}
<a href="?{% url_replace page=public_page_count %}">last &raquo;</a>
{% endif %}
</span>
</div>
+7
View File
@@ -1671,6 +1671,13 @@
return false
}
// A new snapshot has only the synthetic "Browse all files" card.
// Selecting it would replace the live monitor with an empty index.
// Keep progress/empty state until the user explicitly opens a file.
if (document.getElementById('main-frame-wrapper')?.dataset.hasOutputs !== '1') {
return false
}
const selectedCard = document.querySelector('.thumb-card.selected-card') || document.querySelector('.thumb-card')
const selectedLink = selectedCard && (selectedCard.querySelector('a[target=preview]') || selectedCard.querySelector('a'))
if (selectedCard && selectedLink) {
@@ -85,6 +85,7 @@
link.href = '/admin/core/snapshot/?tags__id__exact=' + td.id;
link.className = 'tag-link';
link.textContent = td.name;
link.title = td.name;
pill.appendChild(link);
var removeBtn = document.createElement('button');
+16 -19
View File
@@ -1155,7 +1155,19 @@ body.change-list:not(.model-snapshot) #changelist #toolbar,
opacity: 0.75;
}
#result_list tbody td.field-preview_icon {
position: relative;
}
#result_list tbody td.field-preview_icon > a {
position: absolute;
inset: 4px 2px;
display: block;
}
#result_list tbody td.field-preview_icon .snapshot-thumbnail {
width: 100% !important;
height: 100% !important;
transition: box-shadow 140ms ease, transform 140ms ease;
}
@@ -2226,7 +2238,7 @@ tbody .output-link:hover {opacity: 1;}
.model-snapshot.change-list #result_list tbody th.field-created_at_display > a,
.model-snapshot.change-list #result_list tbody td.field-created_at_display > a {
display: inline-flex;
max-width: none;
max-width: 100%;
white-space: nowrap !important;
}
@@ -2234,8 +2246,10 @@ tbody .output-link:hover {opacity: 1;}
.model-snapshot.change-list #result_list tbody td.field-created_at_display .snapshot-created-at {
display: inline-flex;
flex-direction: row !important;
flex-wrap: wrap;
justify-content: center;
gap: 5px;
row-gap: 0;
white-space: nowrap !important;
}
@@ -2250,13 +2264,10 @@ tbody .output-link:hover {opacity: 1;}
grid-area: preview;
align-self: stretch;
place-items: center;
min-height: 110px;
overflow: hidden;
}
.model-snapshot.change-list #result_list tbody td.field-preview_icon .snapshot-thumbnail {
width: 80px !important;
height: 110px !important;
max-width: 100%;
box-sizing: border-box;
}
@@ -2278,7 +2289,7 @@ tbody .output-link:hover {opacity: 1;}
padding-left: 0;
}
.model-snapshot.change-list #result_list tbody td.field-title_str a {
.model-snapshot.change-list #result_list tbody td.field-title_str > a {
display: block;
max-width: 100%;
overflow-wrap: anywhere;
@@ -2514,26 +2525,12 @@ tbody .output-link:hover {opacity: 1;}
}
.model-snapshot.change-list #result_list tbody tr {
grid-template-columns: 24px 84px minmax(0, 1fr);
gap: 2px 4px;
grid-template-areas:
"check preview date"
"check preview title"
"check preview tags"
"check preview status"
"check preview files"
"check preview size";
}
.model-snapshot.change-list #result_list tbody td.field-files .files-icons {
gap: 0;
}
.model-snapshot.change-list #result_list tbody td.field-status_with_progress,
.model-snapshot.change-list #result_list tbody td.field-size_with_stats {
justify-self: start;
text-align: left;
}
}
body.embedded-change-list #content {
+8 -2
View File
@@ -1663,6 +1663,8 @@ def resolve_abxpkg_binary_env(
deps_from: Path | list[Path] | tuple[Path, ...] | None = None,
) -> dict[str, str]:
"""Resolve required test dependencies through abxpkg's normal install path."""
from abxpkg.config import apply_exec_env
command_env = dict(env) if env is not None else os.environ.copy()
command_env["ABXPKG_LIB_DIR"] = str(lib_dir)
command = [
@@ -1683,7 +1685,11 @@ def resolve_abxpkg_binary_env(
)
assert result.returncode == 0, result.stderr or result.stdout
payload = json.loads(result.stdout)
return {str(key): str(value) for key, value in payload.items()}
# abxpkg env emits prepend/append deltas, not a replacement environment.
# Applying a PATH ending in ':' verbatim hides the caller's abxpkg CLI.
delta = {str(key): str(value) for key, value in payload.items()}
apply_exec_env(delta, command_env)
return {key: command_env[key] for key in delta}
def run_test_hook(
@@ -1781,7 +1787,7 @@ def install_real_binary(
from archivebox.machine.models import Binary, Machine
binary = Binary.objects.create(
machine=machine or Machine.current(refresh=True),
machine=machine or Machine.current(),
name=name,
binproviders=binproviders,
overrides=overrides or {},
@@ -1,4 +1,6 @@
import pytest
import os
import time
from archivebox.crawls.models import Crawl
@@ -15,3 +17,19 @@ def test_basic_success_case_request(client, tmp_path, api_admin_user, api_header
response = client.get(f"/api/v1/crawls/crawl/{crawl.id}/files/folder/basic.txt", **api_headers)
assert response.status_code == 200, response.content
def test_screencast_frame_is_checked_by_image_request(client, api_admin_user, api_headers):
crawl = Crawl.objects.create(urls="https://example.com/screencast", created_by=api_admin_user)
frame = crawl.output_dir / "chrome_screencast" / "latest.jpg"
frame.parent.mkdir(parents=True, exist_ok=True)
frame.write_bytes(b"stored frame")
started_at = time.time()
os.utime(frame, (started_at - 10, started_at - 10))
url = f"/api/v1/crawls/crawl/{crawl.id}/files/chrome_screencast/latest.jpg?after={started_at}"
assert client.get(url, **api_headers).status_code == 404
os.utime(frame, (started_at + 1, started_at + 1))
response = client.get(url, **api_headers)
assert response.status_code == 200
assert b"".join(response.streaming_content) == b"stored frame"
assert client.get(url.replace(str(started_at), "invalid"), **api_headers).status_code == 400
@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
def test_basic_success_case_request(client, tmp_path, api_headers):
machine = Machine.current(refresh=True)
machine = Machine.current()
binary = install_real_binary("python3", machine=machine)
response = client.get("/api/v1/machine/binaries", **api_headers)
@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
def test_basic_success_case_request(client, tmp_path, api_headers):
machine = Machine.current(refresh=True)
machine = Machine.current()
binary = install_real_binary("python3", machine=machine)
response = client.get(f"/api/v1/machine/binary/{binary.id}", **api_headers)
@@ -8,7 +8,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
def test_basic_success_case_request(client, tmp_path, api_headers):
machine = Machine.current(refresh=True)
machine = Machine.current()
binary = install_real_binary("python3", machine=machine)
response = client.get("/api/v1/machine/binary/by-name/python3", **api_headers)
@@ -7,7 +7,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
def test_basic_success_case_request(client, tmp_path, api_headers):
machine = Machine.current(refresh=True)
machine = Machine.current()
response = client.get(f"/api/v1/machine/machine/{machine.id}", **api_headers)
@@ -7,7 +7,7 @@ pytestmark = pytest.mark.django_db(transaction=True)
def test_basic_success_case_request(client, tmp_path, api_headers):
Machine.current(refresh=True)
Machine.current()
response = client.get("/api/v1/machine/machines", **api_headers)
File diff suppressed because it is too large. Load diff
+377
View File
@@ -0,0 +1,377 @@
#!/usr/bin/env python3
"""
Comprehensive tests for archivebox add command.
Verify add creates snapshots in DB, crawls, source files, and archive directories.
"""
import os
import json
import pty
import select
import time
import termios
import signal
import subprocess
import psutil
from pathlib import Path
import pytest
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.conftest import (
cli_env,
run_archivebox_cmd,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
pytestmark = pytest.mark.django_db(transaction=True)
IMPORT_FORMAT_EXPECTATIONS = {
"rss": {
"url": "https://example.com/",
"title": "RSS Example Import",
"date": "2024-01-01",
"tags": {"rss-tag", "metadata"},
},
"netscape": {
"url": "https://www.iana.org/domains/reserved",
"title": "IANA Reserved Domains",
"date": "2024-01-02",
"tags": {"netscape-tag", "metadata"},
},
"dom": {
"url": "https://www.iana.org/help/example-domains",
},
"json": {
"url": "https://example.com/?archivebox-json-import=1",
"title": "JSON Import Example",
"date": "2024-01-03",
"tags": {"json-tag", "metadata"},
},
"jsonl": {
"url": "https://example.com/?archivebox-jsonl-import=1",
"title": "JSONL Import Example",
"date": "2024-01-04",
"tags": {"jsonl-tag", "metadata"},
},
"txt": {
"url": "https://example.org/",
},
}
def write_import_format_files(base_dir: Path) -> dict[str, Path]:
files = {
"rss": base_dir / "test_rss.xml",
"netscape": base_dir / "test_netscape.html",
"dom": base_dir / "test_dom.html",
"json": base_dir / "test_bookmarks.json",
"jsonl": base_dir / "test_bookmarks.jsonl",
"txt": base_dir / "test_urls.txt",
}
files["rss"].write_text(
"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>ArchiveBox RSS import fixture</title>
<link>https://example.com/</link>
<description>ArchiveBox RSS import fixture</description>
<item>
<title>RSS Example Import</title>
<link>https://example.com/</link>
<guid>https://example.com/</guid>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<category>rss-tag</category>
<category>metadata</category>
</item>
</channel>
</rss>
""",
encoding="utf-8",
)
files["netscape"].write_text(
"""<!DOCTYPE NETSCAPE-Bookmark-file-1>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
<TITLE>Bookmarks</TITLE>
<H1>Bookmarks</H1>
<DL><p>
<DT><A HREF="https://www.iana.org/domains/reserved" ADD_DATE="1704153600" TAGS="netscape-tag,metadata">IANA Reserved Domains</A>
</DL><p>
""",
encoding="utf-8",
)
files["dom"].write_text(
"""<!doctype html>
<html>
<head><title>DOM import fixture</title></head>
<body>
<a href="https://www.iana.org/help/example-domains">IANA Example Domains</a>
</body>
</html>
""",
encoding="utf-8",
)
files["json"].write_text(
json.dumps(
{
"url": "https://example.com/?archivebox-json-import=1",
"title": "JSON Import Example",
"tags": ["json-tag", "metadata"],
"bookmarked_at": "2024-01-03T00:00:00+00:00",
},
)
+ "\n",
encoding="utf-8",
)
files["jsonl"].write_text(
json.dumps(
{
"url": "https://example.com/?archivebox-jsonl-import=1",
"title": "JSONL Import Example",
"tags": "jsonl-tag,metadata",
"bookmarked_at": "2024-01-04T00:00:00+00:00",
},
)
+ "\n",
encoding="utf-8",
)
files["txt"].write_text(
"Plain text import fixture containing https://example.org/ as a real live URL.\n",
encoding="utf-8",
)
return files
IMPORT_FORMAT_ENV = {
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"PLUGINS": "parse_html_urls,parse_jsonl_urls,parse_netscape_urls,parse_rss_urls,parse_txt_urls",
"SAVE_WGET": "False",
"SAVE_HEADERS": "False",
"USE_CHROME": "False",
"URL_ALLOWLIST": r"example\.com|example\.org|iana\.org|www\.iana\.org",
}
def assert_expected_import_snapshots(cwd: Path, expected_urls: set[str]) -> None:
allowed_statuses = {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
with use_archivebox_db(cwd):
rows = list(Snapshot.objects.filter(url__in=expected_urls).values_list("url", "status"))
counts = {url: 0 for url in expected_urls}
bad_statuses = []
for url, status in rows:
counts[url] += 1
if status not in allowed_statuses:
bad_statuses.append((url, status))
assert all(count == 1 for count in counts.values()), counts
assert not bad_statuses, bad_statuses
def malicious_add_inputs(tmp_path: Path, *, safe_url: str) -> tuple[list[str], Path]:
other_crawl_source = tmp_path / "sources" / "other_crawl_source.txt"
other_crawl_source.parent.mkdir(parents=True, exist_ok=True)
other_crawl_source.write_text("https://example.com/not-owned-by-this-crawl\n", encoding="utf-8")
canary = tmp_path / "archivebox_shell_injection_canary"
return (
[
safe_url,
"file:///etc/hosts",
"/etc/hosts",
"../../../../etc/passwd",
f"file://{other_crawl_source}",
str(other_crawl_source),
f"'; touch {canary}; #",
f'" && touch {canary} && echo "',
f"$(touch {canary})",
f"`touch {canary}`",
"""<?xml version="1.0"?>
<!DOCTYPE rss [
<!ENTITY localfile SYSTEM "file:///etc/hosts">
]>
<rss version="2.0" xmlns:xi="http://www.w3.org/2001/XInclude">
<channel>
<item><title>&localfile;</title><link>file:///etc/passwd</link></item>
<xi:include href="file:///etc/hosts" parse="text"/>
</channel>
</rss>""",
],
canary,
)
def assert_no_file_or_shell_payload_snapshots(cwd: Path, *, canary: Path) -> None:
with use_archivebox_db(cwd):
snapshots = list(Snapshot.objects.all())
assert not canary.exists()
assert not [snapshot.url for snapshot in snapshots if str(snapshot.url).startswith("file:")]
for forbidden in ("/etc/hosts", "/etc/passwd", "other_crawl_source", "archivebox_shell_injection_canary"):
assert not [snapshot.url for snapshot in snapshots if forbidden in str(snapshot.url)]
@pytest.mark.parametrize(
"choice,plugins",
[(choice, "chrome") for choice in ["skip", "retry", "abort", "ctrl-c", "noninteractive"]] + [("noninteractive", "archivewebpage")],
)
def test_add_interrupts_active_hook(initialized_archive, choice, plugins):
env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True")
# The timed observation below is for a running navigate hook. Resolve the
# selected plugin's real binaries before starting it, so a cold install is
# not counted as a hook-start failure.
installed = run_archivebox_cmd(["install", plugins], cwd=initialized_archive, env=env, timeout=600)
assert installed.returncode == 0, installed.stderr or installed.stdout
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 160))
output = bytearray()
result = None
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
raise AssertionError(output.decode(errors="replace"))
def active_hook():
with use_archivebox_db(initialized_archive):
return Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
status="running",
).first()
try:
result = run_archivebox_cmd(
["add", f"--plugins={plugins}", "https://example.com"],
cwd=initialized_archive,
env=env,
stdin=subprocess.DEVNULL if choice == "noninteractive" else slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
read_until(lambda: active_hook() is not None)
hook = active_hook()
assert hook is not None
result.send_signal(signal.SIGINT)
if choice != "noninteractive":
read_until(lambda: b"Choice [skip]:" in output)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
assert not psutil.pid_exists(hook.pid)
if choice == "retry":
os.write(master, b"r")
read_until(lambda: (new_hook := active_hook()) is not None and new_hook.pid != hook.pid)
prompt_offset = len(output)
result.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output[prompt_offset:])
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
if choice == "ctrl-c":
result.send_signal(signal.SIGINT)
else:
os.write(master, b"a" if choice == "abort" else b"\r")
read_until(lambda: result.poll() is not None)
assert result.returncode == (130 if choice in {"abort", "ctrl-c", "noninteractive"} else 0), output.decode(errors="replace")
if choice == "noninteractive":
assert b"Choice [skip]:" not in output
assert not psutil.pid_exists(hook.pid)
assert b"Traceback" not in output
with use_archivebox_db(initialized_archive):
assert not ArchiveResult.objects.filter(hook_name="on_Snapshot__30_chrome_navigate").exists()
if plugins == "archivewebpage":
start = ArchiveResult.objects.get(hook_name="on_Snapshot__16_archivewebpage_start")
assert start.status == ArchiveResult.StatusChoices.NORESULTS
assert not ArchiveResult.objects.filter(hook_name="on_Snapshot__65_archivewebpage_stop").exists()
assert not ArchiveResult.objects.filter(plugin="archivewebpage", status="succeeded").exists()
assert not any(output["name"] == "archivewebpage" for output in start.snapshot.get_html_details_context()["archiveresults"])
for log in (initialized_archive / "logs").glob("worker_runner_add_*.log"):
assert "Choice [skip]:" not in log.read_text()
finally:
if result is not None and result.poll() is None:
result.terminate()
result.wait(timeout=15)
os.close(slave)
os.close(master)
def test_add_single_url_records_url_in_crawl(initialized_archive):
"""Test that adding a single URL queues a crawl with the submitted URL."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
assert "Crawl.save() outside runner process" not in result.stderr
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.all())
assert crawl.urls == "https://example.com"
assert crawl.get_urls_list() == ["https://example.com"]
assert snapshots == []
def test_add_with_depth_0_flag(initialized_archive):
"""Test that --depth=0 flag is accepted and works."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
assert "unrecognized arguments: --depth" not in result.stderr
def test_add_records_max_url_and_size_limits_on_crawl(initialized_archive):
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
[
"add",
"--index-only",
"--depth=1",
"--max-urls=3",
"--crawl-max-size=45mb",
"--crawl-timeout=120",
"--snapshot-max-size=5mb",
"https://example.com",
],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
columns = {field.name for field in Crawl._meta.local_fields}
with use_archivebox_db(initialized_archive):
config = Crawl.objects.values_list("config", flat=True).get() or {}
assert {"max_urls", "crawl_max_size", "crawl_timeout", "snapshot_max_size"}.isdisjoint(columns)
assert config["CRAWL_MAX_URLS"] == 3
assert config["CRAWL_MAX_SIZE"] == 45 * 1024 * 1024
assert config["CRAWL_TIMEOUT"] == 120
assert config["SNAPSHOT_MAX_SIZE"] == 5 * 1024 * 1024
def test_add_without_args_shows_usage(initialized_archive):
"""Test that add without URLs fails with a usage hint instead of crashing."""
result = run_archivebox_cmd(
["add"],
)
combined = result.stdout + result.stderr
assert result.returncode != 0
assert "usage" in combined.lower() or "url" in combined.lower()
+422
View File
@@ -0,0 +1,422 @@
#!/usr/bin/env python3
"""
Comprehensive tests for archivebox add command.
Verify add creates snapshots in DB, crawls, source files, and archive directories.
"""
import os
import json
import pty
import select
import time
import termios
from pathlib import Path
import pytest
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.tests.conftest import (
cli_env,
run_archivebox_cmd,
run_queued_crawls,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
from .test_cli_add_1 import (
pytestmark as pytestmark,
IMPORT_FORMAT_EXPECTATIONS as IMPORT_FORMAT_EXPECTATIONS,
write_import_format_files as write_import_format_files,
IMPORT_FORMAT_ENV as IMPORT_FORMAT_ENV,
assert_expected_import_snapshots as assert_expected_import_snapshots,
malicious_add_inputs as malicious_add_inputs,
assert_no_file_or_shell_payload_snapshots as assert_no_file_or_shell_payload_snapshots,
)
@pytest.mark.parametrize("terminal_stream", ["stdout", "stderr"])
def test_add_renders_live_progress_in_foreground_terminal(initialized_archive, terminal_stream):
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 160))
output = bytearray()
result = None
try:
result = run_archivebox_cmd(
["add", "--plugins=parse_txt_urls", "https://example.com"],
cwd=initialized_archive,
env=cli_env(USE_COLOR="True", SHOW_PROGRESS="True"),
**{terminal_stream: slave},
wait=False,
start_new_session=True,
)
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
elif result.poll() is not None:
break
assert result.poll() == 0, output.decode(errors="replace")
assert b"\x1b[?25l" in output, output.decode(errors="replace")
assert b"\x1b[?25h" in output, output.decode(errors="replace")
assert b"on_Snapshot__71_parse_txt_urls" in output
assert b"\x1b[2K" in output
with use_archivebox_db(initialized_archive):
assert Crawl.objects.get().status == Crawl.StatusChoices.SEALED
snapshot = Snapshot.objects.get()
assert snapshot.url == "https://example.com"
parsed_urls = Path(snapshot.output_dir) / "parse_txt_urls" / "urls.jsonl"
assert json.loads(parsed_urls.read_text())["url"] == "https://iana.org/domains/example"
finally:
if result is not None and result.poll() is None:
result.terminate()
result.wait(timeout=10)
os.close(slave)
os.close(master)
def test_add_redirected_progress_remains_plain_text(initialized_archive):
result = run_archivebox_cmd(
["add", "--plugins=parse_txt_urls", "https://example.com"],
cwd=initialized_archive,
env=cli_env(),
)
assert result.returncode == 0, result.stderr
assert "on_Snapshot__71_parse_txt_urls" in result.stderr
assert "\x1b[?25l" not in result.stdout + result.stderr
@pytest.mark.timeout(360)
def test_add_stdin_import_formats_preserve_metadata_and_crawl_inner_urls(initialized_archive):
"""`archivebox add < import-file` should normalize rich import formats before crawling URLs."""
import_files = write_import_format_files(initialized_archive)
expected_urls = {case["url"] for case in IMPORT_FORMAT_EXPECTATIONS.values()}
env = cli_env(**IMPORT_FORMAT_ENV)
for import_path in import_files.values():
source_text = import_path.read_text(encoding="utf-8")
result = run_archivebox_cmd(
["add", "--bg", "--depth=0", "--tag=cli-stdin-import"],
cwd=initialized_archive,
env=env,
stdin=source_text,
timeout=360,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.order_by("-created_at").first()
assert crawl is not None
assert crawl.snapshot_set.count() == 0
assert crawl.urls == source_text
run_queued_crawls(initialized_archive, env=env, timeout=240)
with use_archivebox_db(initialized_archive):
for crawl in Crawl.objects.all():
assert not crawl.snapshot_set.filter(url__startswith="archivebox://").exists()
root_input = (crawl.output_dir / "input" / "staticfile" / "stdin.txt").read_text(encoding="utf-8")
assert root_input == crawl.urls
assert_expected_import_snapshots(initialized_archive, expected_urls)
list_result = run_archivebox_cmd(
["list", "--json"],
cwd=initialized_archive,
env=env,
timeout=60,
)
assert list_result.returncode == 0, list_result.stderr or list_result.stdout
for expected_url in expected_urls:
assert expected_url in list_result.stdout
with use_archivebox_db(initialized_archive):
crawls = list(Crawl.objects.order_by("created_at"))
snapshots_by_url = {snapshot.url: snapshot for snapshot in Snapshot.objects.prefetch_related("tags").filter(url__in=expected_urls)}
tags_by_url = {snapshot.url: set(snapshot.tags.values_list("name", flat=True)) for snapshot in snapshots_by_url.values()}
assert len(crawls) == len(import_files)
assert [crawl.urls for crawl in crawls] == [path.read_text(encoding="utf-8") for path in import_files.values()]
assert all(crawl.tags_str == "cli-stdin-import" for crawl in crawls)
assert all(crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED} for crawl in crawls)
assert len(snapshots_by_url) == len(expected_urls)
for import_name, expected in IMPORT_FORMAT_EXPECTATIONS.items():
snapshot = snapshots_by_url.get(expected["url"])
assert snapshot is not None, f"{import_name} did not create Snapshot for {expected['url']}"
assert snapshot.status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
if expected.get("title"):
assert snapshot.title == expected["title"]
if expected.get("date"):
assert snapshot.bookmarked_at.date().isoformat() == expected["date"]
if expected.get("tags"):
assert expected["tags"] | {"cli-stdin-import"} <= tags_by_url[snapshot.url]
def test_add_bg_queues_direct_url_snapshot(initialized_archive):
"""Background add queues explicit URL arguments as real URL snapshots."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--bg", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.all())
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
assert crawl.urls == "https://example.com"
assert snapshots == []
@pytest.mark.timeout(180)
def test_add_title_hook_env_gets_canonical_runtime_config_without_archivebox_selectors_or_aliases(initialized_archive):
env = os.environ.copy()
env.update(
{
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"PLUGINS": "title",
"SAVE_TITLE": "True",
"SECRET_KEY": "hook-env-secret-must-not-leak",
"PUBLIC_ADD_VIEW": "True",
"ADMIN_PASSWORD": "hook-env-admin-password-must-not-leak",
"URL_BLACKLIST": "$^",
"ARCHIVEBOX_TEST_ARBITRARY_ENV": "preserved-user-env",
"LD_PRELOAD": "",
"TIMEOUT": "60",
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1",
},
)
result = run_archivebox_cmd(
[
"add",
"--depth=0",
"--plugins=title",
"https://example.com/?archivebox-title-env=1",
],
cwd=initialized_archive,
env=env,
timeout=180,
)
assert result.returncode == 0, result.stderr or result.stdout
assert "Crawl.save() outside runner process" not in result.stdout + result.stderr
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
title_result = ArchiveResult.objects.select_related("process").get(snapshot=snapshot, plugin="title")
process_env = title_result.process.env
assert crawl.config["PLUGINS"] == "title"
assert snapshot.url == "https://example.com/?archivebox-title-env=1"
assert snapshot.status == Snapshot.StatusChoices.SEALED
assert snapshot.title == "Example Domain"
assert title_result.status == ArchiveResult.StatusChoices.SUCCEEDED
assert title_result.output_str == "Example Domain"
assert title_result.process.exit_code == 0
assert process_env["TITLE_ENABLED"] == "True"
assert process_env["CHROME_ENABLED"] == "True"
assert process_env["WGET_ENABLED"] == "False"
assert process_env["ARCHIVEBOX_TEST_ARBITRARY_ENV"] == "preserved-user-env"
assert process_env["LD_PRELOAD"] == ""
assert "PLUGINS" not in process_env
assert "SAVE_TITLE" not in process_env
assert "SEARCH_BACKEND_ENGINE" not in process_env
assert "SECRET_KEY" not in process_env
assert "ADMIN_PASSWORD" not in process_env
assert "PUBLIC_ADD_VIEW" not in process_env
assert "URL_BLACKLIST" not in process_env
assert "hook-env-secret-must-not-leak" not in json.dumps(process_env)
assert "hook-env-admin-password-must-not-leak" not in json.dumps(process_env)
def test_add_creates_crawl_record(initialized_archive):
"""Test that add command creates a Crawl record in the database."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
with use_archivebox_db(initialized_archive):
crawl_count = Crawl.objects.count()
assert crawl_count == 1
def test_add_rejects_file_path_argument(initialized_archive):
"""Local files must be piped through stdin, not passed as archiveable path arguments."""
env = cli_env(disable_extractors=True)
urls_file = initialized_archive / "urls.txt"
urls_file.write_text("https://example.com\nhttps://example.org\n")
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", str(urls_file)],
cwd=initialized_archive,
env=env,
)
assert result.returncode != 0
with use_archivebox_db(initialized_archive):
assert Crawl.objects.count() == 0
assert Snapshot.objects.count() == 0
assert "No URLs provided" in (result.stderr or result.stdout)
def test_add_records_url_filter_overrides_on_crawl(initialized_archive):
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
[
"add",
"--index-only",
"--depth=0",
"--domain-allowlist=example.com,*.example.com",
"--domain-denylist=static.example.com",
"https://example.com",
],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
assert crawl.config["URL_ALLOWLIST"] == "example.com,*.example.com"
assert crawl.config["URL_DENYLIST"] == "static.example.com"
assert not (initialized_archive / "personas" / "Default" / "chrome_extensions").exists()
def test_add_help_shows_depth_and_tag_options(initialized_archive):
"""Test that add --help documents the main filter and crawl options."""
result = run_archivebox_cmd(
["add", "--help"],
)
assert result.returncode == 0
assert "--depth" in result.stdout
assert "--max-urls" in result.stdout
assert "--crawl-max-size" in result.stdout
assert "--crawl-timeout" in result.stdout
assert "--snapshot-max-size" in result.stdout
assert "--tag" in result.stdout
def test_add_index_only_creates_direct_url_snapshot(initialized_archive):
"""Index-only add seeds explicit URL args for the runner to snapshot."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
run_queued_crawls(initialized_archive, env)
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot = Snapshot.objects.get()
assert crawl.urls == "https://example.com"
assert snapshot.url == "https://example.com"
assert snapshot.depth == 0
def test_snapshot_create_sets_snapshot_timestamp(initialized_archive):
"""Test the user-facing snapshot creation path sets a timestamp."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["snapshot", "create", "https://example.com"],
cwd=initialized_archive,
env=env,
check=True,
)
with use_archivebox_db(initialized_archive):
timestamp = Snapshot.objects.values_list("timestamp", flat=True).get()
assert timestamp is not None
assert len(str(timestamp)) > 0
@pytest.mark.timeout(180)
def test_cli_recursive_crawl_processes_discovered_html_urls(initialized_archive, recursive_test_site):
env = os.environ.copy()
env.update(
{
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
"TIMEOUT": "60",
"SAVE_WGET": "true",
"SAVE_HEADERS": "false",
"SAVE_TITLE": "false",
"SAVE_READABILITY": "false",
"SAVE_SINGLEFILE": "false",
"SAVE_MERCURY": "false",
"SAVE_SCREENSHOT": "false",
"SAVE_PDF": "false",
"SAVE_DOM": "false",
"SAVE_ARCHIVEDOTORG": "false",
"SAVE_GIT": "false",
"SAVE_YTDLP": "false",
"SAVE_FAVICON": "false",
"PARSE_HTML_URLS_ENABLED": "true",
"PARSE_DOM_OUTLINKS_ENABLED": "false",
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
},
)
root_url = recursive_test_site["root_url"]
child_url = recursive_test_site["child_urls"][0]
result = run_archivebox_cmd(
[
"add",
"--depth=2",
"--max-urls=2",
"--crawl-max-size=50mb",
"--tag=recursive-flow",
"--parser=url_list",
"--plugins=wget,parse_html_urls",
root_url,
],
cwd=initialized_archive,
env=env,
timeout=180,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()
snapshots = list(Snapshot.objects.order_by("depth", "url").values_list("url", "depth", "status"))
archive_results = list(
ArchiveResult.objects.select_related("snapshot")
.order_by("snapshot__depth", "snapshot__url", "plugin")
.values_list("snapshot__url", "plugin", "status", "output_files"),
)
assert crawl[0] == 2
assert crawl[1] == "recursive-flow"
crawl_config = crawl[2] or {}
assert crawl_config["CRAWL_MAX_URLS"] == 2
assert crawl_config["CRAWL_MAX_SIZE"] == 50 * 1024 * 1024
assert crawl_config.get("SNAPSHOT_MAX_SIZE", 0) == 0
assert (root_url, 0, "sealed") in snapshots
assert any(url == child_url and depth == 1 and status == "sealed" for url, depth, status in snapshots)
by_url_plugin = {(url, plugin): status for url, plugin, status, _files in archive_results}
assert by_url_plugin[(root_url, "wget")] == "succeeded"
assert by_url_plugin[(root_url, "parse_html_urls")] == "succeeded"
assert by_url_plugin[(child_url, "wget")] == "succeeded"
urls_outputs = list((initialized_archive / "archive/users/system/snapshots").rglob("parse_html_urls/urls.jsonl"))
assert urls_outputs
assert any(child_url in path.read_text() for path in urls_outputs)
+456
View File
@@ -0,0 +1,456 @@
#!/usr/bin/env python3
"""
Comprehensive tests for archivebox add command.
Verify add creates snapshots in DB, crawls, source files, and archive directories.
"""
import os
import pty
import select
import time
import termios
import signal
import re
import subprocess
import psutil
from pathlib import Path
import pytest
from django.utils import timezone
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.conftest import (
cli_env,
run_archivebox_cmd,
run_queued_crawls,
resolve_abxpkg_chrome_env,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
from .test_cli_add_1 import (
pytestmark as pytestmark,
IMPORT_FORMAT_EXPECTATIONS as IMPORT_FORMAT_EXPECTATIONS,
write_import_format_files as write_import_format_files,
IMPORT_FORMAT_ENV as IMPORT_FORMAT_ENV,
assert_expected_import_snapshots as assert_expected_import_snapshots,
malicious_add_inputs as malicious_add_inputs,
assert_no_file_or_shell_payload_snapshots as assert_no_file_or_shell_payload_snapshots,
)
@pytest.mark.parametrize("choice", ["skip", "retry", "ctrl-c", "double", "noninteractive"])
def test_add_interrupts_background_only_capture(initialized_archive, choice):
"""Neither pause nor abort/reclaim semantics may depend on a foreground hook."""
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 200))
output = bytearray()
result = None
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.05)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
raise AssertionError(output.decode(errors="replace"))
def active_hook():
with use_archivebox_db(initialized_archive):
return Process.objects.filter(archiveresult__plugin="forumdl", status="running").order_by("-started_at").first()
try:
result = run_archivebox_cmd(
["add", "--depth=1", "--plugins=forumdl", "https://news.ycombinator.com"],
cwd=initialized_archive,
env=cli_env(),
stdin=subprocess.DEVNULL if choice == "noninteractive" else slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
read_until(lambda: active_hook() is not None)
hook = active_hook()
children = psutil.Process(result.pid).children(recursive=True)
result.send_signal(signal.SIGINT)
if choice == "double":
read_until(lambda: b"Forwarding Ctrl+C" in output)
result.send_signal(signal.SIGINT)
elif choice != "noninteractive":
read_until(lambda: b"Choice [skip]:" in output, timeout=20)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
assert not psutil.pid_exists(hook.pid)
if choice == "retry":
os.write(master, b"r")
read_until(lambda: (next_hook := active_hook()) is not None and next_hook.pid != hook.pid)
children.extend(psutil.Process(result.pid).children(recursive=True))
offset = len(output)
result.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output[offset:], timeout=20)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
os.write(master, b"\r" if choice == "skip" else b"\x03")
read_until(lambda: result.poll() is not None, timeout=30)
assert result.returncode == (0 if choice == "skip" else 130), output.decode(errors="replace")
assert not [child.pid for child in children if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
assert not psutil.pid_exists(hook.pid)
assert b"Traceback" not in output
if choice == "noninteractive":
assert b"Choice [skip]:" not in output
# An explicit abort must not be mistaken for takeover and restart work.
with use_archivebox_db(initialized_archive):
assert Process.objects.filter(worker_type="worker_runner", process_type="orchestrator").count() == 1
if choice != "skip":
assert b"Aborting crawl" in output
finally:
if result is not None and result.poll() is None:
result.terminate()
result.wait(timeout=20)
os.close(slave)
os.close(master)
def test_background_completions_render_above_interrupt_prompt(initialized_archive, recursive_test_site):
env = cli_env(WGET_ARGS_EXTRA='["--limit-rate=20"]', INFINISCROLL_SCROLL_DELAY="10000", CHROME_HEADLESS="True")
# The timed observation is for a running infinite-scroll hook. Resolve its
# real browser and wget dependencies before the readiness window begins.
installed = run_archivebox_cmd(
["install", "wget", "chrome"],
cwd=initialized_archive,
env=env,
timeout=600,
)
assert installed.returncode == 0, installed.stderr or installed.stdout
env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env))
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 200))
output = bytearray()
result = None
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
for log in sorted(initialized_archive.rglob("on_Snapshot__*.*.log")):
print(f"\n{log.relative_to(initialized_archive)}:\n{log.read_text(errors='replace')}")
raise AssertionError(output.decode(errors="replace"))
def result_for(plugin):
with use_archivebox_db(initialized_archive):
return ArchiveResult.objects.filter(plugin=plugin).first()
try:
result = run_archivebox_cmd(
["add", "--depth=1", "--plugins=wget,infiniscroll,parse_html_urls", recursive_test_site["root_url"]],
cwd=initialized_archive,
# The small test-owned page still transfers slowly enough for wget
# to finish while the real foreground hook's prompt is open.
env=env,
stdin=slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
read_until(lambda: (scroll := result_for("infiniscroll")) is not None and scroll.status == "started")
result.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
before_completion = len(output)
read_until(lambda: (wget := result_for("wget")) is not None and wget.status == "succeeded")
read_until(lambda: b"Choice [skip]:" in output[before_completion:])
screen_updates = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output[before_completion:].decode(errors="replace"))
completed_at = screen_updates.index("on_Snapshot__35_wget")
prompt_at = screen_updates.rindex("Interrupted on_Snapshot__45_infiniscroll")
assert "succeeded" in screen_updates[completed_at:prompt_at], screen_updates
assert completed_at < prompt_at < screen_updates.rindex("Choice [skip]:"), screen_updates
assert result_for("parse_html_urls") is None
(initialized_archive / "interrupt-terminal.txt").write_bytes(output)
os.write(master, b"a")
read_until(lambda: result.poll() is not None)
assert result.returncode == 130, output.decode(errors="replace")
assert b"Traceback" not in output
finally:
if result is not None and result.poll() is None:
result.terminate()
result.wait(timeout=15)
os.close(slave)
os.close(master)
def test_add_summary_points_to_snapshot_outputs(initialized_archive):
result = run_archivebox_cmd(
["add", "--plugins=parse_txt_urls", "https://example.com"],
cwd=initialized_archive,
env=cli_env(),
)
assert result.returncode == 0, result.stderr
with use_archivebox_db(initialized_archive):
snapshot = Snapshot.objects.get()
assert Path(snapshot.output_dir).is_dir()
assert str(snapshot.output_dir) in result.stdout
assert "snapshot output saved to:" in result.stdout
assert "crawl logs and setup:" in result.stdout
assert "crawl output saved to:" not in result.stdout
def test_initial_crawl_creation_does_not_warn_as_an_outside_runner_update(initialized_archive, caplog):
from archivebox.core.takeover_util import current_command
with use_archivebox_db(initialized_archive):
command = current_command(Process.TypeChoices.ADD, data_dir=initialized_archive)
caplog.clear()
with caplog.at_level("WARNING", logger="archivebox.workers.models"):
crawl = Crawl.objects.create(urls="https://example.com", status=Crawl.StatusChoices.QUEUED)
assert "Crawl.save() outside runner process" not in caplog.text
caplog.clear()
with caplog.at_level("WARNING", logger="archivebox.workers.models"):
crawl.save(update_fields=["modified_at"])
command.mark_exited(exit_code=0)
assert "Crawl.save() outside runner process" in caplog.text
@pytest.mark.timeout(240)
def test_add_rejects_file_path_and_shell_injection_payloads(initialized_archive):
"""CLI add must not turn user-supplied local paths or shell payloads into snapshots."""
safe_url = "https://example.com/?archivebox-cli-security=1"
inputs, canary = malicious_add_inputs(initialized_archive, safe_url=safe_url)
env = cli_env(**IMPORT_FORMAT_ENV)
result = run_archivebox_cmd(
["add", "--bg", "--depth=0", "--tag=cli-security"],
cwd=initialized_archive,
env=env,
stdin="\n".join(inputs),
timeout=120,
)
assert result.returncode == 0, result.stderr or result.stdout
run_queued_crawls(initialized_archive, env=env, timeout=120)
assert_expected_import_snapshots(initialized_archive, {safe_url})
assert_no_file_or_shell_payload_snapshots(initialized_archive, canary=canary)
with use_archivebox_db(initialized_archive):
snapshots = list(Snapshot.objects.filter(url=safe_url).values_list("url", "status", "tags__name"))
crawl = Crawl.objects.get()
assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
assert len({url for url, _status, _tag in snapshots}) == 1
assert all(
status in {Snapshot.StatusChoices.QUEUED, Snapshot.StatusChoices.STARTED, Snapshot.StatusChoices.SEALED}
for _url, status, _tag in snapshots
)
assert "cli-security" in {tag for _url, _status, tag in snapshots}
@pytest.mark.timeout(180)
def test_run_rejects_depth_two_file_url_snapshot_injected_directly_with_db_update(initialized_archive):
"""A queued Snapshot row with a file:// URL must not be allowed to run hooks or recurse."""
secret_url = "https://example.com/?archivebox-sql-depth2-file-secret=1"
local_source = initialized_archive / "not_owned_by_depth_two_snapshot.txt"
local_source.write_text(f"{secret_url}\n", encoding="utf-8")
file_url = local_source.resolve().as_uri()
env = cli_env(**{**IMPORT_FORMAT_ENV, "PLUGINS": "parse_txt_urls", "SAVE_HEADERS": "False", "SAVE_WGET": "False"})
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.create(
urls="https://example.com/?archivebox-sql-depth2-root=1",
max_depth=2,
tags_str="sql-depth2-file-url",
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
root_snapshot = Snapshot.objects.create(
url="https://example.com/?archivebox-sql-depth2-root=1",
crawl=crawl,
depth=0,
status=Snapshot.StatusChoices.SEALED,
retry_at=None,
)
injected_snapshot = Snapshot.objects.create(
url="https://example.com/?archivebox-sql-depth2-placeholder=1",
crawl=crawl,
parent_snapshot=root_snapshot,
depth=2,
status=Snapshot.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
Snapshot.objects.filter(pk=injected_snapshot.pk).update(url=file_url)
result = run_archivebox_cmd(
["run", f"--snapshot-id={injected_snapshot.id}"],
cwd=initialized_archive,
env=env,
timeout=180,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
injected_snapshot.refresh_from_db()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
file_results = list(
ArchiveResult.objects.filter(
snapshot=injected_snapshot,
).values_list("plugin", "status"),
)
assert injected_snapshot.url == file_url
assert secret_url not in snapshot_urls
assert file_results == []
def test_add_index_only_rejected_urls_leave_empty_crawl_for_runner_to_seal(initialized_archive):
"""Index-only add only creates the crawl; rejected URLs are sealed by the runner."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
[
"add",
"--index-only",
"--depth=0",
"--url-denylist=example.com",
"https://example.com",
],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert snapshot_urls == set()
run_queued_crawls(initialized_archive, env)
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert snapshot_urls == set()
def test_add_index_only_rejects_archivebox_internal_urls(initialized_archive):
"""Index-only add must apply the same internal URL guard as snapshot creation."""
env = cli_env(disable_extractors=True)
internal_urls = [
"http://archivebox.localhost:9292/admin/",
"http://web.archivebox.localhost:9292/",
"http://api.archivebox.localhost:9292/api/v1/docs",
"http://snap-2fb8e923c58c.archivebox.localhost:9292/index.html",
]
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", *internal_urls],
cwd=initialized_archive,
env={**env, "BASE_URL": "http://archivebox.localhost:9292"},
)
assert result.returncode == 0
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
assert crawl.urls.splitlines() == internal_urls
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert snapshot_urls == set()
def test_add_multiple_urls_single_command(initialized_archive):
"""Test adding multiple URLs in a single command records one crawl."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com", "https://example.org"],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.order_by("url").values_list("url", "depth"))
assert crawl.urls.splitlines() == ["https://example.com", "https://example.org"]
assert snapshots == []
def test_add_with_tags(initialized_archive):
"""Test adding URL with tags stores tags_str in crawl.
With --index-only, Tag objects are not created until archiving happens.
Tags are stored as a string in the Crawl.tags_str field.
"""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "--tag=test,example", "https://example.com"],
cwd=initialized_archive,
env=env,
)
with use_archivebox_db(initialized_archive):
tags_str = Crawl.objects.values_list("tags_str", flat=True).get()
# Tags are stored as a comma-separated string in crawl
assert "test" in tags_str or "example" in tags_str
def test_add_with_overwrite_flag(initialized_archive):
"""Test that --overwrite flag forces re-archiving."""
env = cli_env(disable_extractors=True)
# Add URL first time
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
# Add with overwrite
result = run_archivebox_cmd(
["add", "--index-only", "--overwrite", "https://example.com"],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
assert "unrecognized arguments: --overwrite" not in result.stderr
def test_add_index_only_queues_crawl_without_starting_runner(initialized_archive):
"""Test that --index-only creates only a queued crawl and returns fast."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
timeout=30, # Should be fast
)
assert result.returncode == 0
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.all())
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is None
assert crawl.urls == "https://example.com"
assert snapshots == []
+511
View File
@@ -0,0 +1,511 @@
#!/usr/bin/env python3
"""
Comprehensive tests for archivebox add command.
Verify add creates snapshots in DB, crawls, source files, and archive directories.
"""
import os
import json
import pty
import select
import time
import termios
import signal
import re
import psutil
from pathlib import Path
import pytest
from django.utils import timezone
from archivebox.core.models import ArchiveResult, Snapshot, SnapshotTag
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.conftest import (
cli_env,
find_snapshot_dir,
run_archivebox_cmd,
run_queued_crawls,
resolve_abxpkg_chrome_env,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
from .test_cli_add_1 import (
pytestmark as pytestmark,
IMPORT_FORMAT_EXPECTATIONS as IMPORT_FORMAT_EXPECTATIONS,
write_import_format_files as write_import_format_files,
IMPORT_FORMAT_ENV as IMPORT_FORMAT_ENV,
assert_expected_import_snapshots as assert_expected_import_snapshots,
malicious_add_inputs as malicious_add_inputs,
assert_no_file_or_shell_payload_snapshots as assert_no_file_or_shell_payload_snapshots,
)
def test_abort_stops_long_running_background_hook(initialized_archive):
env = cli_env(INFINISCROLL_SCROLL_DELAY="10000", CHROME_HEADLESS="True")
# Keep first-use dependency installation outside the timed hook-start
# observation. This test is about interrupting a running hook; a cold
# browser or forumdl install must not consume its readiness window.
installed = run_archivebox_cmd(
["install", "forumdl", "chrome"],
cwd=initialized_archive,
env=env,
timeout=600,
)
assert installed.returncode == 0, installed.stderr or installed.stdout
env.update(resolve_abxpkg_chrome_env(Path(env["ABXPKG_LIB_DIR"]), env))
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 200))
output = bytearray()
result = None
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
raise AssertionError(output.decode(errors="replace"))
def result_for(plugin):
with use_archivebox_db(initialized_archive):
return ArchiveResult.objects.filter(plugin=plugin).first()
try:
result = run_archivebox_cmd(
["add", "--depth=1", "--plugins=forumdl,infiniscroll", "https://news.ycombinator.com"],
cwd=initialized_archive,
env=env,
stdin=slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
try:
read_until(lambda: (scroll := result_for("infiniscroll")) is not None and scroll.status == "started")
except AssertionError:
# Capture state before teardown terminates the runner: otherwise a
# missed readiness deadline loses the install/hook failure itself.
with use_archivebox_db(initialized_archive):
print("Hook readiness:", list(ArchiveResult.objects.values("plugin", "status", "output_str")))
print("Process readiness:", list(Process.objects.values("process_type", "status", "exit_code", "stderr")))
raise
read_until(lambda: "running pid=" in re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", output.decode(errors="replace")))
result.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
forum = result_for("forumdl")
assert forum is not None and forum.status == "started"
children = psutil.Process(result.pid).children(recursive=True)
abort_offset = len(output)
os.write(master, b"\x03")
read_until(lambda: b"Aborting crawl" in output[abort_offset:], timeout=3)
read_until(lambda: result.poll() is not None, timeout=25)
assert result.returncode == 130, output.decode(errors="replace")
assert not [child.pid for child in children if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
assert any("INFO:root:GET" in log.read_text() for log in initialized_archive.rglob("on_Snapshot__33_forumdl.*.stderr.log"))
assert b"Stopped during crawl abort" in output
assert b"INFO:root:GET" not in output[abort_offset:]
assert b"Traceback" not in output
finally:
if result is not None and result.poll() is None:
result.terminate()
result.wait(timeout=15)
os.close(slave)
os.close(master)
def test_add_bg_persists_queue_without_starting_extraction_workers(initialized_archive):
result = run_archivebox_cmd(
["add", "--bg", "--plugins=wget", "--crawl-max-concurrent-snapshots=4", "https://example.com", "https://example.org"],
cwd=initialized_archive,
env=cli_env(),
)
assert result.returncode == 0, result.stderr
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
assert crawl.status == Crawl.StatusChoices.QUEUED
assert crawl.retry_at is not None
assert crawl.get_urls_list() == ["https://example.com", "https://example.org"]
assert crawl.config["PLUGINS"] == "wget"
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 4
assert not Snapshot.objects.exists()
assert not ArchiveResult.objects.exists()
assert not Process.objects.filter(process_type__in=[Process.TypeChoices.ORCHESTRATOR, Process.TypeChoices.HOOK]).exists()
@pytest.mark.timeout(180)
def test_run_rejects_file_url_injected_directly_into_crawl_urls_with_db_update(initialized_archive):
"""Runner must validate Crawl.urls again when DB writes bypass normal add/create paths."""
secret_url = "https://example.com/?archivebox-sql-crawl-file-secret=1"
local_source = initialized_archive / "not_owned_by_crawl_urls.txt"
local_source.write_text(f"{secret_url}\n", encoding="utf-8")
file_url = local_source.resolve().as_uri()
env = cli_env(**{**IMPORT_FORMAT_ENV, "PLUGINS": "parse_txt_urls", "SAVE_HEADERS": "False", "SAVE_WGET": "False"})
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.create(
urls="https://example.com/?archivebox-sql-crawl-control=1",
max_depth=2,
tags_str="sql-file-url",
status=Crawl.StatusChoices.QUEUED,
retry_at=timezone.now(),
)
bad_jsonl = json.dumps({"type": "Snapshot", "url": file_url, "depth": 0, "tags": "sql-file-url"})
Crawl.objects.filter(pk=crawl.pk).update(urls=bad_jsonl)
result = run_archivebox_cmd(
["run", f"--crawl-id={crawl.id}"],
cwd=initialized_archive,
env=env,
timeout=180,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
crawl.refresh_from_db()
snapshot_urls = set(Snapshot.objects.values_list("url", flat=True))
assert crawl.status in {Crawl.StatusChoices.STARTED, Crawl.StatusChoices.SEALED}
assert file_url not in snapshot_urls
assert secret_url not in snapshot_urls
@pytest.mark.timeout(180)
def test_add_tagged_single_url_seals_without_duplicate_snapshot_tags(initialized_archive):
env = os.environ.copy()
env.update(
{
"USE_COLOR": "False",
"SHOW_PROGRESS": "False",
"PLUGINS": "parse_txt_urls,title",
"TIMEOUT": "60",
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "1",
},
)
result = run_archivebox_cmd(
[
"add",
"--bg",
"--depth=0",
"--tag=tagged-single-url",
"https://example.com/?archivebox-tagged-single-url=1",
],
cwd=initialized_archive,
env=env,
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
run_queued_crawls(initialized_archive, env, timeout=180)
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
snapshots = list(Snapshot.objects.order_by("depth", "url"))
tag_counts = {
snapshot.url: SnapshotTag.objects.filter(snapshot=snapshot, tag__name="tagged-single-url").count() for snapshot in snapshots
}
results = list(
ArchiveResult.objects.select_related("snapshot")
.order_by("snapshot__depth", "snapshot__url", "plugin")
.values_list("snapshot__url", "plugin", "status", "output_str"),
)
assert crawl.status == Crawl.StatusChoices.SEALED
assert [(snapshot.url, snapshot.depth, snapshot.status) for snapshot in snapshots] == [
("https://example.com/?archivebox-tagged-single-url=1", 0, Snapshot.StatusChoices.SEALED),
]
assert tag_counts == {
"https://example.com/?archivebox-tagged-single-url=1": 1,
}
by_url_plugin = {(url, plugin): status for url, plugin, status, _output in results}
assert by_url_plugin[("https://example.com/?archivebox-tagged-single-url=1", "title")] == "succeeded"
unexpected_failures = [(url, plugin, status, output) for url, plugin, status, output in results if status == "failed"]
assert not unexpected_failures
def test_add_direct_url_creates_snapshot_without_internal_input_file(initialized_archive):
"""Explicit URL args materialize real snapshots when the runner claims the crawl."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
run_queued_crawls(initialized_archive, env)
with use_archivebox_db(initialized_archive):
snapshot = Snapshot.objects.get()
assert snapshot.url == "https://example.com"
assert snapshot.depth == 0
assert not (snapshot.output_dir / "staticfile" / "stdin.txt").exists()
def test_add_with_depth_1_flag(initialized_archive):
"""Test that --depth=1 flag is accepted."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--index-only", "--depth=1", "https://example.com"],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
assert "unrecognized arguments: --depth" not in result.stderr
def test_add_rejects_invalid_depth_values(initialized_archive):
"""Test that add rejects depth values outside the supported range."""
env = cli_env(disable_extractors=True)
for depth in ("5", "-1"):
result = run_archivebox_cmd(
["add", "--index-only", f"--depth={depth}", "https://example.com"],
cwd=initialized_archive,
env=env,
)
stderr = result.stderr.lower()
assert result.returncode != 0
assert "invalid" in stderr or "not one of" in stderr
def test_add_records_selected_persona_on_crawl(initialized_archive):
"""Test add persists the selected persona so browser config derives from it later."""
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", "--persona=Default", "https://example.com"],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get()
assert crawl.persona_id
assert "ACTIVE_PERSONA" not in crawl.config
assert (initialized_archive / "personas" / "Default" / "chrome_profile").is_dir()
def test_add_duplicate_url_creates_separate_crawls(initialized_archive):
"""Test that adding the same URL twice creates separate crawls.
Each 'add' command creates a new Crawl. Multiple crawls can archive the same URL.
This allows re-archiving URLs at different times.
"""
env = cli_env(disable_extractors=True)
# Add URL first time
run_archivebox_cmd(
["add", "--index-only", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
# Add same URL second time with --update to opt out of ONLY_NEW.
run_archivebox_cmd(
["add", "--index-only", "--update", "--depth=0", "https://example.com"],
cwd=initialized_archive,
env=env,
)
run_queued_crawls(initialized_archive, env)
with use_archivebox_db(initialized_archive):
crawl_count = Crawl.objects.count()
snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "depth"))
# Each add creates a new crawl with its own queued work.
assert crawl_count == 2
assert snapshots == [("https://example.com", 0), ("https://example.com", 0)]
def test_snapshot_create_creates_current_output_directory(initialized_archive):
"""Test the user-facing snapshot creation path creates an output directory."""
env = cli_env(disable_extractors=True)
run_archivebox_cmd(
["snapshot", "create", "https://example.com"],
cwd=initialized_archive,
env=env,
check=True,
)
with use_archivebox_db(initialized_archive):
snapshot_id = str(Snapshot.objects.values_list("id", flat=True).get())
snapshot_dir = find_snapshot_dir(initialized_archive, snapshot_id)
assert snapshot_dir is not None, f"Snapshot output directory not found for {snapshot_id}"
assert snapshot_dir.is_dir()
def test_add_preserves_apostrophe_in_snapshot_url(initialized_archive):
url = "https://aaib.gov.in/What's%20New%20Assets/Preliminary%20Report%20VT-EXO.pdf"
env = cli_env(disable_extractors=True)
result = run_archivebox_cmd(
["add", "--index-only", "--depth=0", url],
cwd=initialized_archive,
env=env,
)
assert result.returncode == 0, result.stderr
run_queued_crawls(initialized_archive, env)
with use_archivebox_db(initialized_archive):
assert Crawl.objects.get().urls == url
assert Snapshot.objects.get().url == url
@pytest.mark.timeout(180)
def test_cli_add_real_urls_with_options_writes_inspectable_outputs(initialized_archive):
wget_urls = [
"https://example.com",
"https://pirate.github.io/stress-tests/challenge.html",
]
chrome_url = "https://example.com/?archivebox-chrome-flow=1"
env = os.environ.copy()
env.pop("CHROME_BINARY", None)
env.update(
{
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
"TIMEOUT": "60",
"SAVE_WGET": "true",
"SAVE_HEADERS": "false",
"SAVE_TITLE": "false",
"SAVE_READABILITY": "false",
"SAVE_SINGLEFILE": "false",
"SAVE_MERCURY": "false",
"SAVE_SCREENSHOT": "false",
"SAVE_PDF": "false",
"SAVE_DOM": "false",
"SAVE_ARCHIVEDOTORG": "false",
"SAVE_GIT": "false",
"SAVE_YTDLP": "false",
"SAVE_FAVICON": "false",
},
)
_cmd_result = run_archivebox_cmd(
[
"add",
"--depth=0",
"--max-urls=2",
"--crawl-max-size=10mb",
"--tag=real-flow,challenge",
"--parser=url_list",
"--plugins=wget",
*wget_urls,
],
cwd=initialized_archive,
env=env,
timeout=180,
)
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert returncode == 0, stderr or stdout
chrome_env = env | {
"SAVE_WGET": "false",
"SAVE_HEADERS": "true",
"SAVE_TITLE": "true",
"CHROME_HEADLESS": "true",
"CHROME_SANDBOX": "false",
"CHROME_ISOLATION": "snapshot",
}
_cmd_result = run_archivebox_cmd(
["install", "chrome"],
cwd=initialized_archive,
env=chrome_env,
timeout=600,
)
install_stdout, install_stderr, install_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert install_returncode == 0, install_stderr or install_stdout
chrome_env.update(resolve_abxpkg_chrome_env(Path(chrome_env["ABXPKG_LIB_DIR"]), chrome_env))
_cmd_result = run_archivebox_cmd(
[
"add",
"--depth=0",
"--max-urls=1",
"--crawl-max-size=10mb",
"--tag=chrome-flow",
"--parser=url_list",
"--plugins=chrome,wget,headers,title",
chrome_url,
],
cwd=initialized_archive,
env=chrome_env,
timeout=180,
)
chrome_stdout, chrome_stderr, chrome_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert chrome_returncode == 0, chrome_stderr or chrome_stdout
_cmd_result = run_archivebox_cmd(
["list", "--tag=real-flow"],
cwd=initialized_archive,
env=env,
timeout=60,
)
list_stdout, list_stderr, list_returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert list_returncode == 0, list_stderr or list_stdout
listed = [json.loads(line) for line in list_stdout.splitlines() if line.strip()]
assert {item["url"] for item in listed} >= set(wget_urls)
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.order_by("-created_at").values_list("max_depth", "tags_str", "config").first()
real_flow_crawl = Crawl.objects.filter(tags_str="real-flow,challenge").values_list("max_depth", "tags_str", "config").first()
snapshots = list(Snapshot.objects.order_by("url").values_list("id", "url", "depth", "status", "title"))
archive_results = list(
ArchiveResult.objects.select_related("snapshot")
.order_by("snapshot__url", "plugin")
.values_list("snapshot__url", "plugin", "status", "output_files", "output_size", "output_str"),
)
processes = list(Process.objects.filter(process_type="hook").values_list("process_type", "status", "exit_code", "pwd", "cmd"))
assert real_flow_crawl is not None
assert real_flow_crawl[0] == 0
assert real_flow_crawl[1] == "real-flow,challenge"
real_flow_config = real_flow_crawl[2] or {}
assert real_flow_config["CRAWL_MAX_URLS"] == 2
assert real_flow_config["CRAWL_MAX_SIZE"] == 10 * 1024 * 1024
assert real_flow_config.get("SNAPSHOT_MAX_SIZE", 0) == 0
assert "wget" in real_flow_config["PLUGINS"]
assert crawl is not None
assert crawl[1] == "chrome-flow"
assert "wget,headers,title" in json.dumps(crawl[2] or {})
snapshot_urls = {url for _id, url, _depth, _status, _title in snapshots}
assert snapshot_urls >= {*wget_urls, chrome_url}
assert all(depth == 0 for _id, url, depth, _status, _title in snapshots)
by_url_plugin = {(url, plugin): status for url, plugin, status, _files, _size, _output in archive_results}
assert by_url_plugin[("https://example.com", "wget")] == "succeeded"
assert by_url_plugin[("https://pirate.github.io/stress-tests/challenge.html", "wget")] == "succeeded"
assert by_url_plugin[(chrome_url, "headers")] == "succeeded"
assert by_url_plugin[(chrome_url, "title")] == "succeeded"
unexpected_results = [
(url, plugin, status, output) for url, plugin, status, _files, _size, output in archive_results if status != "succeeded"
]
assert not unexpected_results
snapshot_root = initialized_archive / "archive/users/system/snapshots"
html_outputs = [path for path in snapshot_root.rglob("wget/**/*.html") if path.is_file()]
header_outputs = [path for path in snapshot_root.rglob("headers/**/headers.json") if path.is_file() and path.stat().st_size > 0]
title_outputs = [path for path in snapshot_root.rglob("title/title.txt") if path.is_file() and path.stat().st_size > 0]
index_outputs = [path for path in snapshot_root.rglob("index.jsonl") if path.is_file()]
assert html_outputs
assert header_outputs
assert any("example.com" in path.read_text(errors="ignore").lower() for path in header_outputs)
assert title_outputs
assert any("Example Domain" in path.read_text(errors="ignore") for path in title_outputs)
assert len(index_outputs) >= len(wget_urls) + 1
combined_html = "\n".join(path.read_text(errors="ignore") for path in html_outputs)
assert "Example Domain" in combined_html
assert "Browser Agent Challenge for AI Browser Drivers" in combined_html
assert processes
assert any("wget" in (pwd or "") or "wget" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
assert any("headers" in (pwd or "") or "headers" in (cmd or "") for _type, _status, _exit, pwd, cmd in processes)
+1
View File
@@ -269,6 +269,7 @@ def test_init_recovers_from_pre_squash_dev_history(tmp_path):
("core", "0021_auto_20220724_1254"),
("core", "0021_auto_20221128_1116"),
("core", "0023_alter_archiveresult_options_archiveresult_abid_and_more"),
("core", "0056_progress_covering_indexes"),
("core", "0074_alter_snapshot_downloaded_at"),
("core", "0075_crawl"),
("core", "0075_archiveresult_retry_at"),
File diff suppressed because it is too large. Load diff
+177
View File
@@ -0,0 +1,177 @@
"""
Tests for archivebox run CLI command.
Tests cover:
- run with stdin JSONL (Crawl, Snapshot, ArchiveResult)
- create-or-update behavior (records with/without id)
- pass-through output (for chaining)
"""
import os
import pty
import select
import signal
import subprocess
import termios
import time
import psutil
import pytest
from archivebox.tests.conftest import (
cli_env,
create_test_url,
parse_jsonl_output,
run_archivebox_cmd,
)
RUN_TEST_ENV = {
"PLUGINS": "favicon",
"SAVE_FAVICON": "True",
}
def _install_real_chrome_for_test(data_dir, env, *, isolation):
env["CHROME_ISOLATION"] = isolation
env["CHROME_HEADLESS"] = "true"
env["CHROME_SANDBOX"] = "false"
install_process = run_archivebox_cmd(
["install", "chrome"],
cwd=data_dir,
env=env,
timeout=600,
)
assert install_process.returncode == 0, install_process.stderr or install_process.stdout
@pytest.mark.django_db(transaction=True)
@pytest.mark.parametrize(
"choice,target",
[(choice, "crawl") for choice in ["skip", "retry", "abort", "ctrl-c", "noninteractive"]]
+ [("skip", "snapshot"), ("skip", "global"), ("early-skip", "crawl")]
+ [(choice, "daemon") for choice in ["daemon-tty", "daemon-noninteractive"]]
+ [("daemon-term", "daemon")],
)
def test_run_crawl_interrupts_active_hook(initialized_archive, choice, target):
"""Foreground run uses the same terminal interrupt choices as add."""
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True", PLUGINS="chrome")
# This test starts its 45-second observation at the active navigate hook.
# Install Chrome through the real CLI first so a cold binary download is
# not mistaken for a hook that never started.
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], cwd=initialized_archive, env=env)
assert created.returncode == 0, created.stdout + created.stderr
snapshot = next(record for record in parse_jsonl_output(created.stdout) if record.get("type") == "Snapshot")
crawl_id = snapshot["crawl_id"]
run_args = {
"crawl": [f"--crawl-id={crawl_id}"],
"snapshot": [f"--snapshot-id={snapshot['id']}"],
"global": [],
"daemon": ["--daemon", f"--crawl-id={crawl_id}"],
}[target]
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 160))
output = bytearray()
result = None
owned_processes = []
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
raise AssertionError(output.decode(errors="replace"))
def active_hook():
with use_archivebox_db(initialized_archive):
return Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
status="running",
).first()
try:
result = run_archivebox_cmd(
["run", *run_args],
cwd=initialized_archive,
env=env,
stdin=subprocess.DEVNULL if choice in {"noninteractive", "daemon-noninteractive"} else slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
read_until(lambda: active_hook() is not None)
first_hook = active_hook()
assert first_hook is not None
with use_archivebox_db(initialized_archive):
hook_pids = Process.objects.filter(status="running", process_type="hook").values_list("pid", flat=True)
for pid in hook_pids:
try:
process = psutil.Process(pid)
owned_processes.extend([process, *process.children(recursive=True)])
except psutil.NoSuchProcess:
continue
result.send_signal(signal.SIGTERM if choice == "daemon-term" else signal.SIGINT)
if choice not in {"noninteractive", "daemon-tty", "daemon-noninteractive", "daemon-term"}:
read_until(lambda: b"Choice [skip]:" in output, timeout=30)
if choice != "early-skip":
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
if choice == "retry":
os.write(master, b"r")
read_until(lambda: (next_hook := active_hook()) is not None and next_hook.pid != first_hook.pid)
before_prompt = len(output)
result.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output[before_prompt:], timeout=30)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
if choice == "ctrl-c":
result.send_signal(signal.SIGINT)
else:
os.write(master, b"a" if choice == "abort" else b"\r")
read_until(lambda: result.poll() is not None, timeout=45)
expected_exit = 143 if choice == "daemon-term" else (0 if choice in {"skip", "early-skip", "retry"} else 130)
assert result.returncode == expected_exit, output.decode(errors="replace")
assert not psutil.pid_exists(first_hook.pid)
_, live_processes = psutil.wait_procs(owned_processes, timeout=5)
assert not [process.pid for process in live_processes if process.is_running() and process.status() != psutil.STATUS_ZOMBIE]
assert b"Traceback" not in output, output.decode(errors="replace")
with use_archivebox_db(initialized_archive):
crawl_after = Crawl.objects.get(id=crawl_id)
snapshot_after = Snapshot.objects.get(id=snapshot["id"])
if choice == "daemon-term":
# A supervisor stop displaces this runner; the user's crawl
# and snapshot remain resumable for the next owner.
assert crawl_after.status == Crawl.StatusChoices.STARTED
assert snapshot_after.status == Snapshot.StatusChoices.STARTED
assert crawl_after.retry_at is not None and snapshot_after.retry_at is not None
else:
assert crawl_after.status == Crawl.StatusChoices.SEALED
assert snapshot_after.status == Snapshot.StatusChoices.SEALED
assert not Process.objects.filter(status="running", pid__isnull=False).exists()
if choice in {"noninteractive", "daemon-tty", "daemon-noninteractive", "daemon-term"}:
assert b"Choice [skip]:" not in output
finally:
if result is not None and result.poll() is None:
result.terminate()
try:
result.wait(timeout=20)
except subprocess.TimeoutExpired:
result.send_signal(signal.SIGKILL)
result.wait(timeout=5)
for process in owned_processes:
try:
if process.is_running() and process.status() != psutil.STATUS_ZOMBIE:
process.terminate()
except psutil.NoSuchProcess:
pass
psutil.wait_procs(owned_processes, timeout=3)
os.close(slave)
os.close(master)
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+489
View File
@@ -0,0 +1,489 @@
"""
Tests for archivebox run CLI command.
Tests cover:
- run with stdin JSONL (Crawl, Snapshot, ArchiveResult)
- create-or-update behavior (records with/without id)
- pass-through output (for chaining)
"""
import os
import pty
import select
import signal
import subprocess
import termios
import time
import psutil
import pytest
from archivebox.tests.conftest import (
cleanup_process_group,
cli_env,
create_test_url,
parse_jsonl_output,
pid_is_alive,
run_archivebox_cmd,
wait_for_log,
wait_for_pid_to_disappear,
)
from .test_cli_run_1 import (
RUN_TEST_ENV as RUN_TEST_ENV,
_install_real_chrome_for_test as _install_real_chrome_for_test,
)
@pytest.mark.django_db(transaction=True)
@pytest.mark.parametrize("choice", ["typed", "signal"])
def test_run_third_interrupt_forces_chosen_abort(initialized_archive, choice):
"""Direct run exits at once when Ctrl+C follows an explicit abort choice."""
from archivebox.machine.models import Process
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(CHROME_DELAY_AFTER_LOAD="60", CHROME_TIMEOUT="120", CHROME_HEADLESS="True", PLUGINS="chrome")
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], cwd=initialized_archive, env=env)
assert created.returncode == 0, created.stdout + created.stderr
crawl_id = next(record["crawl_id"] for record in parse_jsonl_output(created.stdout) if record.get("type") == "Snapshot")
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 160))
output = bytearray()
result = None
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
raise AssertionError(output.decode(errors="replace"))
def active_hook_pid():
with use_archivebox_db(initialized_archive):
hook = Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
status="running",
).first()
return hook.pid if hook is not None and hook.is_running else None
try:
result = run_archivebox_cmd(
["run", f"--crawl-id={crawl_id}"],
cwd=initialized_archive,
env=env,
stdin=slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
read_until(lambda: active_hook_pid() is not None)
hook_pid = active_hook_pid()
assert hook_pid is not None
owned = psutil.Process(result.pid).children(recursive=True)
result.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
if choice == "typed":
os.write(master, b"a")
else:
result.send_signal(signal.SIGINT)
read_until(lambda: b"Aborting crawl" in output, timeout=5)
forced_at = time.monotonic()
result.send_signal(signal.SIGINT)
read_until(lambda: result.poll() is not None, timeout=5)
assert time.monotonic() - forced_at < 2.0
assert result.returncode == 130, output.decode(errors="replace")
assert not pid_is_alive(hook_pid)
assert not [child.pid for child in owned if child.is_running() and child.status() != psutil.STATUS_ZOMBIE]
assert b"Traceback" not in output
finally:
if result is not None and result.poll() is None:
result.terminate()
result.wait(timeout=20)
os.close(slave)
os.close(master)
@pytest.mark.django_db(transaction=True)
def test_cli_run_retries_snapshot_after_filesystem_failure(initialized_archive):
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(live=True, PLUGINS="hashes")
created = run_archivebox_cmd(["snapshot", "create", create_test_url()], cwd=initialized_archive, env=env)
assert created.returncode == 0, created.stdout + created.stderr
snapshot_id = next(record["id"] for record in parse_jsonl_output(created.stdout) if record.get("type") == "Snapshot")
with use_archivebox_db(initialized_archive):
snapshot = Snapshot.objects.get(pk=snapshot_id)
crawl_id = str(snapshot.crawl_id)
directory = snapshot.output_dir
directory.mkdir(parents=True, exist_ok=True)
(directory / "input.txt").write_text("Capture survives a temporary filesystem failure")
broken_directory = directory / "hashes"
broken_directory.symlink_to("hashes")
failed = run_archivebox_cmd(["run", f"--snapshot-id={snapshot_id}"], cwd=initialized_archive, env=env, timeout=60)
assert failed.returncode == 1, failed.stdout + failed.stderr
assert "FileExistsError: [Errno 17]" in failed.stderr
with use_archivebox_db(initialized_archive):
snapshot = Snapshot.objects.get(pk=snapshot_id)
assert snapshot.status == Snapshot.StatusChoices.STARTED
assert snapshot.retry_at is not None
assert not ArchiveResult.objects.filter(snapshot=snapshot).exists()
broken_directory.unlink()
resumed = run_archivebox_cmd(["run", f"--crawl-id={crawl_id}"], cwd=initialized_archive, env=env, timeout=60)
assert resumed.returncode == 0, resumed.stdout + resumed.stderr
with use_archivebox_db(initialized_archive):
snapshot = Snapshot.objects.get(pk=snapshot_id)
assert snapshot.status == Snapshot.StatusChoices.SEALED
result = snapshot.archiveresult_set.get(plugin="hashes")
assert result.status == ArchiveResult.StatusChoices.SUCCEEDED
assert result.output_size > 0
assert (directory / "hashes").is_dir()
@pytest.mark.django_db(transaction=True)
@pytest.mark.parametrize("precreate_snapshot", [False, True])
def test_newest_crawl_runs_before_older_queued_and_started_crawls(initialized_archive, precreate_snapshot):
from datetime import timedelta
from django.utils import timezone
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(live=True, PLUGINS="hashes")
old_url, queued_url, newest_url = create_test_url(), create_test_url(), create_test_url()
old = run_archivebox_cmd(["snapshot", "create", old_url], cwd=initialized_archive, env=env)
assert old.returncode == 0, old.stdout + old.stderr
with use_archivebox_db(initialized_archive):
snapshot = Snapshot.objects.get(url=old_url)
Crawl.objects.filter(pk=snapshot.crawl_id).update(status=Crawl.StatusChoices.STARTED)
Snapshot.objects.filter(pk=snapshot.pk).update(retry_at=timezone.now() - timedelta(days=1))
queued = run_archivebox_cmd(
["snapshot" if precreate_snapshot else "crawl", "create", queued_url],
cwd=initialized_archive,
env=env,
)
assert queued.returncode == 0, queued.stdout + queued.stderr
newest = run_archivebox_cmd(
["snapshot" if precreate_snapshot else "crawl", "create", newest_url],
cwd=initialized_archive,
env=env,
)
assert newest.returncode == 0, newest.stdout + newest.stderr
result = run_archivebox_cmd(["run", "--no-stdin"], cwd=initialized_archive, env=env, timeout=60)
assert result.returncode == 0, result.stdout + result.stderr
assert all(url in result.stdout for url in (old_url, queued_url, newest_url))
assert result.stdout.index(newest_url) < result.stdout.index(queued_url) < result.stdout.index(old_url), result.stdout
with use_archivebox_db(initialized_archive):
assert (
Snapshot.objects.filter(
url__in=[old_url, queued_url, newest_url],
status=Snapshot.StatusChoices.SEALED,
).count()
== 3
)
@pytest.mark.django_db(transaction=True)
@pytest.mark.timeout(660)
def test_due_snapshots_share_one_crawl_scoped_chrome(initialized_archive, recursive_test_site):
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false")
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
urls = [recursive_test_site["root_url"], recursive_test_site["child_urls"][0]]
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.create(
urls="\n".join(urls),
config={"PLUGINS": "chrome", "CHROME_ISOLATION": "crawl"},
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
)
for url in urls:
Snapshot.objects.create(url=url, crawl=crawl, status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now())
crawl_id = str(crawl.id)
result = run_archivebox_cmd(["run", "--no-stdin", f"--crawl-id={crawl_id}"], cwd=initialized_archive, env=env, timeout=600)
assert result.returncode == 0, result.stdout + result.stderr
with use_archivebox_db(initialized_archive):
snapshots = Snapshot.objects.filter(crawl_id=crawl_id, url__in=urls, status=Snapshot.StatusChoices.SEALED)
assert snapshots.count() == 2
assert all((snapshot.output_dir / "chrome" / "navigation.json").is_file() for snapshot in snapshots)
launches = Process.objects.filter(
pwd__contains=crawl_id,
process_type=Process.TypeChoices.HOOK,
cmd__0__endswith="on_CrawlSetup__90_chrome_launch.daemon.bg.js",
)
assert launches.count() == 1, [(process.pid, process.exit_code) for process in launches]
assert launches.get().exit_code == 0
@pytest.mark.django_db(transaction=True)
def test_busy_crawl_does_not_report_snapshot_progress(initialized_archive):
from datetime import timedelta
from django.utils import timezone
from archivebox.base_models.models import get_or_create_system_user_pk
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import _run_due_snapshot_id
from archivebox.tests.test_orm_helpers import use_archivebox_db
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.create(
urls="https://example.com/",
config={"PLUGINS": "chrome", "CHROME_ISOLATION": "crawl"},
created_by_id=get_or_create_system_user_pk(),
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now() + timedelta(minutes=1),
)
snapshot = Snapshot.objects.create(url=crawl.urls, crawl=crawl, retry_at=timezone.now())
assert _run_due_snapshot_id(str(snapshot.id), lock_seconds=60, interactive_interrupts=False, runtime_config=None) is False
@pytest.mark.django_db(transaction=True)
@pytest.mark.timeout(660)
def test_cli_run_signal_cleans_real_chrome_hook_process_group(initialized_archive, recursive_test_site):
from archivebox.core.models import Snapshot
from archivebox.tests.test_orm_helpers import use_archivebox_db
env = cli_env(live=True, PLUGINS="chrome", CHROME_ISOLATION="crawl", CHROME_HEADLESS="true", CHROME_SANDBOX="false")
_install_real_chrome_for_test(initialized_archive, env, isolation="crawl")
result = run_archivebox_cmd(
["snapshot", "create", recursive_test_site["root_url"]],
cwd=initialized_archive,
env=env,
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
records = parse_jsonl_output(result.stdout)
snapshot_id = next(record["id"] for record in records if record.get("type") == "Snapshot")
with use_archivebox_db(initialized_archive):
browser_state = Snapshot.objects.get(id=snapshot_id).output_dir / "chrome" / "browser.json"
run_log = initialized_archive / "run-signal-chrome.log"
run_log_handle = run_log.open("w", encoding="utf-8")
run_process = run_archivebox_cmd(
["run", f"--snapshot-id={snapshot_id}"],
cwd=initialized_archive,
env=env,
stdout=run_log_handle,
stderr=subprocess.STDOUT,
start_new_session=True,
wait=False,
)
run_log_handle.close()
try:
wait_for_log(browser_state, '"ready": true', timeout=120)
child_pids = [child.pid for child in psutil.Process(run_process.pid).children(recursive=True) if pid_is_alive(child.pid)]
assert child_pids
run_process.send_signal(signal.SIGTERM)
run_process.wait(timeout=30)
output = run_log.read_text(encoding="utf-8", errors="replace")
assert "Runner error" not in output
for pid in child_pids:
wait_for_pid_to_disappear(pid, timeout=15)
finally:
cleanup_process_group(run_process.pid)
class TestRunWithArchiveResult:
"""Tests for `archivebox run` with ArchiveResult input."""
@pytest.mark.django_db(transaction=True)
def test_run_treats_no_id_archiveresult_as_parent_snapshot_plugin_request(self, initialized_archive):
import json
from archivebox.core.models import ArchiveResult
from archivebox.tests.test_orm_helpers import use_archivebox_db
create_result = run_archivebox_cmd(
["snapshot", "create", create_test_url()],
cwd=initialized_archive,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
snapshot_id = next(record["id"] for record in parse_jsonl_output(create_result.stdout) if record.get("type") == "Snapshot")
missing_hook = "on_Snapshot__99_missing_favicon_hook"
request = json.dumps(
{
"type": "ArchiveResult",
"snapshot_id": snapshot_id,
"plugin": "favicon",
"hook_name": missing_hook,
"status": "queued",
},
)
result = run_archivebox_cmd(
["run"],
stdin=f"{request}\n",
cwd=initialized_archive,
timeout=120,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
rows = list(
ArchiveResult.objects.filter(snapshot_id=snapshot_id, plugin="favicon").values_list(
"hook_name",
"status",
"output_str",
),
)
assert len(rows) == 1
assert rows[0][0] != missing_hook
assert rows[0][0].startswith("on_Snapshot__")
assert rows[0][1] in ArchiveResult.FINAL_STATES
def test_run_requeues_failed_archiveresult(self, initialized_archive):
"""Run uses a failed ArchiveResult as a parent Snapshot/plugin reference."""
url = create_test_url()
# Create snapshot and archive result
_cmd_result = run_archivebox_cmd(
["snapshot", "create", url],
cwd=initialized_archive,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
stdout1, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
_cmd_result = run_archivebox_cmd(
["archiveresult", "create", "--plugin=favicon"],
stdin=stdout1,
cwd=initialized_archive,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
stdout2, _, _ = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(stdout2))
initial_run = run_archivebox_cmd(
["run"],
stdin=stdout2,
cwd=initialized_archive,
timeout=120,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
assert initial_run.returncode == 0, initial_run.stderr
persisted_result = run_archivebox_cmd(
["archiveresult", "list", "--plugin=favicon"],
cwd=initialized_archive,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
assert persisted_result.returncode == 0, persisted_result.stderr
assert any(record.get("type") == "ArchiveResult" for record in parse_jsonl_output(persisted_result.stdout))
# Update to failed
update_result = run_archivebox_cmd(
["archiveresult", "update", "--status=failed"],
stdin=persisted_result.stdout,
cwd=initialized_archive,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
assert update_result.returncode == 0, update_result.stderr
failed_result = run_archivebox_cmd(
["archiveresult", "list", "--plugin=favicon"],
cwd=initialized_archive,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
assert failed_result.returncode == 0, failed_result.stderr
failed_records = [record for record in parse_jsonl_output(failed_result.stdout) if record.get("type") == "ArchiveResult"]
assert len(failed_records) == 1
assert failed_records[0]["status"] == "failed"
failed_jsonl = next(line for line in failed_result.stdout.splitlines() if failed_records[0]["id"] in line) + "\n"
# Now run should re-queue it
_cmd_result = run_archivebox_cmd(
["run"],
stdin=failed_jsonl,
cwd=initialized_archive,
timeout=120,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
stdout3, _stderr, code = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert code == 0
records = parse_jsonl_output(stdout3)
ar_records = [r for r in records if r.get("type") == "ArchiveResult"]
assert len(ar_records) >= 1
class TestRunMixedInput:
"""Tests for `archivebox run` with mixed record types."""
def test_run_handles_mixed_records_emitted_by_cli(self, initialized_archive):
"""Run handles real Crawl, Snapshot, and Tag records from CLI stages."""
tag_result = run_archivebox_cmd(
["tag", "create", "mixed-run-tag"],
cwd=initialized_archive,
default_cli_env=True,
disable_extractors=True,
)
assert tag_result.returncode == 0, tag_result.stderr
crawl_result = run_archivebox_cmd(
["crawl", "create", create_test_url()],
cwd=initialized_archive,
default_cli_env=True,
disable_extractors=True,
)
assert crawl_result.returncode == 0, crawl_result.stderr
snapshot_result = run_archivebox_cmd(
["snapshot", "create"],
stdin=crawl_result.stdout,
cwd=initialized_archive,
default_cli_env=True,
disable_extractors=True,
)
assert snapshot_result.returncode == 0, snapshot_result.stderr
result = run_archivebox_cmd(
["run"],
stdin=tag_result.stdout + snapshot_result.stdout,
cwd=initialized_archive,
timeout=120,
env=RUN_TEST_ENV,
default_cli_env=True,
disable_extractors=True,
)
assert result.returncode == 0
records = parse_jsonl_output(result.stdout)
types = {record.get("type") for record in records}
assert {"Crawl", "Snapshot", "Tag"}.issubset(types)
+12 -5
View File
@@ -30,6 +30,7 @@ from archivebox.tests.conftest import (
run_archivebox_cmd,
start_archivebox_server,
stop_archivebox_process,
wait_for_log,
wait_for_log_count,
wait_for_pid_to_disappear,
)
@@ -588,7 +589,7 @@ def test_live_server_machine_search_engine_update_reaches_subsequent_snapshot_ru
"from archivebox.crawls.models import Crawl;"
"from archivebox.core.models import Snapshot;"
"from archivebox.machine.models import Machine;"
"machine = Machine.current(refresh=True);"
"machine = Machine.current();"
"machine.config = {**dict(machine.config or {}), 'SEARCH_BACKEND_ENGINE': 'sqlite'};"
"machine.save(update_fields=['config', 'modified_at']);"
"crawl = Crawl.objects.create("
@@ -813,26 +814,32 @@ def test_live_server_signal_exit_and_resume_uses_existing_supervisor_state(initi
try:
server = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}.log", env=env)
server_log = server.log_path
# HTTP readiness can precede the foreground tail scope. Exercise its
# actual signal policy, where interrupting Popen.poll stranded reaping.
wait_for_log(server_log, "Tailing worker logs (Ctrl+C to stop)...")
os.kill(server.pid, stop_signal)
server.wait(timeout=20)
if expected_notice:
log_text = server_log.read_text(encoding="utf-8", errors="replace")
assert expected_notice in log_text
assert "ArchiveBox server shut down gracefully" in log_text
assert server.returncode == 0, log_text
assert expected_notice in log_text, log_text
assert "ArchiveBox server shut down gracefully" in log_text, log_text
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
resumed = start_archivebox_server(initialized_archive, port=port, log_name=f"server-{stop_signal.name}-resumed.log", env=env)
resumed_log = resumed.log_path
wait_for_log(resumed_log, "Tailing worker logs (Ctrl+C to stop)...")
result = run_archivebox_cmd(["status"], cwd=initialized_archive, env=env, timeout=60)
assert result.returncode == 0, result.stderr or result.stdout
os.kill(resumed.pid, signal.SIGTERM)
resumed.wait(timeout=20)
resumed_text = resumed_log.read_text(encoding="utf-8", errors="replace")
assert "Got SIGTERM" in resumed_text
assert "ArchiveBox server shut down gracefully" in resumed_text
assert resumed.returncode == 0, resumed_text
assert "Got SIGTERM" in resumed_text, resumed_text
assert "ArchiveBox server shut down gracefully" in resumed_text, resumed_text
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
finally:
for proc in (server, resumed):
+10
View File
@@ -194,6 +194,16 @@ class TestRequiredBinaryConfigHandling:
assert Path(resolved["WGET_BINARY"]).is_absolute()
assert Path(resolved["WGET_BINARY"]).is_file()
runtime_env = {**os.environ, **resolved}
assert set(os.environ["PATH"].split(os.pathsep)).issubset(runtime_env["PATH"].split(os.pathsep))
result = subprocess.run(
["/usr/bin/env", "abxpkg", "--version"],
env=runtime_env,
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, result.stderr
@pytest.mark.django_db(transaction=True)
def test_binary_env_var_name_only_handling(self, hermetic_lib_dir):
+23 -12
View File
@@ -174,21 +174,32 @@ class TestMachineModel:
assert machine1.id == machine2.id
def test_machine_current_refreshes_after_interval(self):
"""Machine.current() should refresh after recheck interval."""
import archivebox.machine.models as models
def test_machine_current_does_not_refresh_after_interval(self):
"""A long-lived process must not turn a cached machine read into writes."""
from django.db import connection
from django.test.utils import CaptureQueriesContext
machine1 = Machine.current()
# Manually expire the cache by modifying modified_at
machine1.modified_at = timezone.now() - timedelta(seconds=MACHINE_RECHECK_INTERVAL + 1)
machine1.save()
models._CURRENT_MACHINE = machine1
with CaptureQueriesContext(connection) as queries:
machine2 = Machine.current()
assert Machine.current() is machine1
assert machine2 is machine1
assert list(queries) == []
machine2 = Machine.current()
def test_machine_config_save_updates_cache_without_reinitializing(self):
from django.db import connection
from django.test.utils import CaptureQueriesContext
# Should have fetched/updated the machine (same GUID)
assert machine1.guid == machine2.guid
cached = Machine.current()
edited = Machine.objects.get(pk=cached.pk)
edited.config = {**edited.config, "USER_AGENT": "read-only-machine-test"}
edited.save(update_fields=["config"])
with CaptureQueriesContext(connection) as queries:
current = Machine.current()
assert current is cached
assert current.config["USER_AGENT"] == "read-only-machine-test"
assert list(queries) == []
def test_machine_current_recreates_stale_cached_row(self):
"""Machine.current() should recreate the cached machine if the row was deleted."""
@@ -284,9 +295,9 @@ class TestMachineModel:
"WGET_BINARY": "/tmp/archivebox-test-missing-wget",
}
machine.save(update_fields=["config"])
models._CURRENT_MACHINE = machine
models._CURRENT_MACHINE = None
refreshed = Machine.current(refresh=True)
refreshed = Machine.current()
# Valid binary paths inside ABXPKG_LIB_DIR survive.
assert refreshed.config.get("CHROME_BINARY") == str(chrome_path)
+194 -293
View File
@@ -189,36 +189,28 @@ def test_migration_preserves_supported_extended_08_metadata(migration_08_data, u
assert migrated_counts == expected_counts
def test_migration_preserves_snapshot_count(migration_08_data):
"""Migration should preserve all snapshots from 0.8.x."""
def test_migration_preserves_08_collection(migration_08_data):
"""One real upgrade preserves every field and relationship in the same collection."""
work_dir, db_path, original_data = migration_08_data
expected_count = len(original_data["snapshots"])
with sqlite3.connect(db_path) as conn:
original_tag_count = conn.execute("SELECT COUNT(*) FROM core_snapshot_tags").fetchone()[0]
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
# migration_preserves_snapshot_count
expected_count = len(original_data["snapshots"])
ok, msg = verify_snapshot_count(db_path, expected_count)
assert ok, msg
def test_migration_preserves_snapshot_urls(migration_08_data):
"""Migration should preserve all snapshot URLs from 0.8.x."""
work_dir, db_path, original_data = migration_08_data
# migration_preserves_snapshot_urls
expected_urls = [s["url"] for s in original_data["snapshots"]]
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
ok, msg = verify_snapshot_urls(db_path, expected_urls)
assert ok, msg
def test_migration_preserves_crawls(migration_08_data):
"""Migration should preserve all Crawl records and create default crawl if needed."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
# migration_preserves_crawls
# Count snapshots with NULL crawl_id in original data
snapshots_without_crawl = sum(1 for s in original_data["snapshots"] if s["crawl_id"] is None)
@@ -230,6 +222,191 @@ def test_migration_preserves_crawls(migration_08_data):
ok, msg = verify_crawl_count(db_path, expected_count)
assert ok, msg
# migration_preserves_snapshot_crawl_links
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check EVERY snapshot has a crawl_id after migration
for snapshot in original_data["snapshots"]:
cursor.execute("SELECT crawl_id FROM core_snapshot WHERE url = ?", (snapshot["url"],))
row = cursor.fetchone()
assert row is not None, f"Snapshot {snapshot['url']} not found after migration"
if snapshot["crawl_id"] is not None:
# Snapshots that had a crawl should keep it
assert row[0] == snapshot["crawl_id"], f"Crawl ID changed for {snapshot['url']}: expected {snapshot['crawl_id']}, got {row[0]}"
else:
# Snapshots without a crawl should now have one (the default crawl)
assert row[0] is not None, f"Snapshot {snapshot['url']} should have been assigned to default crawl but has NULL"
conn.close()
# migration_preserves_tags
ok, msg = verify_tag_count(db_path, len(original_data["tags"]))
assert ok, msg
# migration_preserves_archiveresults
expected_count = len(original_data["archiveresults"])
expected_counts = {}
for result in original_data["archiveresults"]:
status = "succeeded" if result["status"] == "success" else result["status"]
key = (result["extractor"], status)
expected_counts[key] = expected_counts.get(key, 0) + 1
ok, msg = verify_archiveresult_count(db_path, expected_count)
assert ok, msg
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status")
migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()}
cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL")
missing_process_count = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM machine_process")
process_count = cursor.fetchone()[0]
conn.close()
assert migrated_counts == expected_counts
assert missing_process_count == 0
assert process_count == expected_count
# migration_preserves_archiveresult_status
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Get status counts
cursor.execute("SELECT status, COUNT(*) FROM core_archiveresult GROUP BY status")
status_counts = dict(cursor.fetchall())
conn.close()
# Original data has known status distribution: succeeded, failed, skipped
assert "succeeded" in status_counts, "Should have succeeded results"
assert "failed" in status_counts, "Should have failed results"
assert "skipped" in status_counts, "Should have skipped results"
# migration_preserves_snapshot_titles
expected_titles = {s["url"]: s["title"] for s in original_data["snapshots"]}
ok, msg = verify_snapshot_titles(db_path, expected_titles)
assert ok, msg
# migration_preserves_foreign_keys
ok, msg = verify_foreign_keys(db_path)
assert ok, msg
# migration_creates_process_records
# Verify Process records created
expected_count = len(original_data["archiveresults"])
ok, msg = verify_process_migration(db_path, expected_count)
assert ok, msg
# migration_creates_binary_records
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check Binary records exist
cursor.execute("SELECT COUNT(*) FROM machine_binary")
binary_count = cursor.fetchone()[0]
# Should have at least one binary per unique extractor
extractors = {ar["extractor"] for ar in original_data["archiveresults"]}
assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}"
cursor.execute("""
SELECT COUNT(*)
FROM machine_process
WHERE cmd != '[]' AND binary_id IS NULL
""")
missing_binary_count = cursor.fetchone()[0]
assert missing_binary_count == 0
cursor.execute("""
SELECT p.cmd, b.name, b.abspath
FROM machine_process p
JOIN machine_binary b ON p.binary_id = b.id
WHERE p.cmd != '[]'
""")
rows = cursor.fetchall()
assert rows
for cmd_raw, binary_name, binary_abspath in rows:
cmd = json.loads(cmd_raw)
assert binary_name == cmd[0]
assert binary_abspath == cmd[0]
cursor.execute("SELECT COUNT(*) FROM machine_process WHERE iface_id IS NULL")
missing_iface_count = cursor.fetchone()[0]
assert missing_iface_count == 0
conn.close()
# migration_preserves_cmd_data
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check that Process records have cmd arrays
cursor.execute("SELECT cmd FROM machine_process WHERE cmd != '[]'")
cmd_records = cursor.fetchall()
# All Processes should have non-empty cmd (test data has json.dumps([extractor, '--version']))
expected_count = len(original_data["archiveresults"])
assert len(cmd_records) == expected_count, f"Expected {expected_count} Processes with cmd, got {len(cmd_records)}"
conn.close()
# no_duplicate_snapshots_after_migration
# Check for duplicate URLs
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("""
SELECT url, COUNT(*) as cnt FROM core_snapshot
GROUP BY url HAVING cnt > 1
""")
duplicates = cursor.fetchall()
conn.close()
assert len(duplicates) == 0, f"Found duplicate URLs: {duplicates}"
# no_orphaned_archiveresults_after_migration
ok, msg = verify_foreign_keys(db_path)
assert ok, msg
# timestamps_preserved_after_migration
original_timestamps = {s["url"]: s["timestamp"] for s in original_data["snapshots"]}
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT url, timestamp FROM core_snapshot")
migrated_timestamps = {row[0]: row[1] for row in cursor.fetchall()}
conn.close()
for url, original_ts in original_timestamps.items():
assert migrated_timestamps.get(url) == original_ts, f"Timestamp changed for {url}: {original_ts} -> {migrated_timestamps.get(url)}"
# crawl_data_preserved_after_migration
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check each crawl's data is preserved
for crawl in original_data["crawls"]:
cursor.execute("SELECT urls, label, status FROM crawls_crawl WHERE id = ?", (crawl["id"],))
row = cursor.fetchone()
assert row is not None, f"Crawl {crawl['id']} not found after migration"
assert row[0] == crawl["urls"], f"URLs mismatch for crawl {crawl['id']}"
assert row[1] == crawl["label"], f"Label mismatch for crawl {crawl['id']}"
assert row[2] == crawl["status"], f"Status mismatch for crawl {crawl['id']}"
conn.close()
# tag_associations_preserved_after_migration
# Count tag associations after migration
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
migrated_count = cursor.fetchone()[0]
conn.close()
assert migrated_count == original_tag_count, f"Tag associations changed: {original_tag_count} -> {migrated_count}"
def test_legacy_default_crawl_is_sealed_without_a_due_retry(tmp_path):
"""Upgrading a legacy collection must not leave its sealed crawl runnable."""
@@ -343,72 +520,6 @@ def test_forward_migration_normalizes_active_legacy_retry_timestamps(migration_0
assert conn.execute("SELECT retry_at FROM machine_process WHERE id = ?", (process_id,)).fetchone() == (expected_utc_timestamp,)
def test_migration_preserves_snapshot_crawl_links(migration_08_data):
"""Migration should preserve snapshot-to-crawl relationships and assign default crawl to orphans."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check EVERY snapshot has a crawl_id after migration
for snapshot in original_data["snapshots"]:
cursor.execute("SELECT crawl_id FROM core_snapshot WHERE url = ?", (snapshot["url"],))
row = cursor.fetchone()
assert row is not None, f"Snapshot {snapshot['url']} not found after migration"
if snapshot["crawl_id"] is not None:
# Snapshots that had a crawl should keep it
assert row[0] == snapshot["crawl_id"], f"Crawl ID changed for {snapshot['url']}: expected {snapshot['crawl_id']}, got {row[0]}"
else:
# Snapshots without a crawl should now have one (the default crawl)
assert row[0] is not None, f"Snapshot {snapshot['url']} should have been assigned to default crawl but has NULL"
conn.close()
def test_migration_preserves_tags(migration_08_data):
"""Migration should preserve all tags."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
ok, msg = verify_tag_count(db_path, len(original_data["tags"]))
assert ok, msg
def test_migration_preserves_archiveresults(migration_08_data):
"""Migration should preserve ArchiveResult rows and link each one to a Process."""
work_dir, db_path, original_data = migration_08_data
expected_count = len(original_data["archiveresults"])
expected_counts = {}
for result in original_data["archiveresults"]:
status = "succeeded" if result["status"] == "success" else result["status"]
key = (result["extractor"], status)
expected_counts[key] = expected_counts.get(key, 0) + 1
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
ok, msg = verify_archiveresult_count(db_path, expected_count)
assert ok, msg
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT plugin, status, COUNT(*) FROM core_archiveresult GROUP BY plugin, status")
migrated_counts = {(plugin, status): count for plugin, status, count in cursor.fetchall()}
cursor.execute("SELECT COUNT(*) FROM core_archiveresult WHERE process_id IS NULL")
missing_process_count = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM machine_process")
process_count = cursor.fetchone()[0]
conn.close()
assert migrated_counts == expected_counts
assert missing_process_count == 0
assert process_count == expected_count
def test_migration_preserves_users_groups_permissions_api_secrets_and_repeated_init(migration_08_data):
work_dir, db_path, original_data = migration_08_data
@@ -446,26 +557,6 @@ def test_process_metadata_migration_rolls_back_completely_on_malformed_row(migra
assert conn.execute("SELECT COUNT(*) FROM machine_binary").fetchone()[0] == 0
def test_migration_preserves_archiveresult_status(migration_08_data):
"""Migration should preserve archive result status values."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Get status counts
cursor.execute("SELECT status, COUNT(*) FROM core_archiveresult GROUP BY status")
status_counts = dict(cursor.fetchall())
conn.close()
# Original data has known status distribution: succeeded, failed, skipped
assert "succeeded" in status_counts, "Should have succeeded results"
assert "failed" in status_counts, "Should have failed results"
assert "skipped" in status_counts, "Should have skipped results"
def test_status_works_after_migration(migration_08_data):
"""Status command should work after migration."""
work_dir, db_path, original_data = migration_08_data
@@ -506,28 +597,6 @@ def test_search_works_after_migration(migration_08_data):
assert ok, msg
def test_migration_preserves_snapshot_titles(migration_08_data):
"""Migration should preserve all snapshot titles."""
work_dir, db_path, original_data = migration_08_data
expected_titles = {s["url"]: s["title"] for s in original_data["snapshots"]}
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
ok, msg = verify_snapshot_titles(db_path, expected_titles)
assert ok, msg
def test_migration_preserves_foreign_keys(migration_08_data):
"""Migration should maintain foreign key relationships."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
ok, msg = verify_foreign_keys(db_path)
assert ok, msg
@pytest.mark.parametrize("has_snapshot_status", (True, False), ids=("with-status", "without-status"))
def test_migration_preserves_08_timestamp_meanings(migration_08_data, has_snapshot_status):
"""0.8.x already has separated timestamp/bookmarked_at/created_at/downloaded_at fields."""
@@ -652,174 +721,6 @@ def test_version_works_after_migration(migration_08_data):
assert "ArchiveBox" in output or "version" in output.lower(), f"Version output missing expected content: {output[:500]}"
def test_migration_creates_process_records(migration_08_data):
"""Migration should create Process records for all ArchiveResults."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
# Verify Process records created
expected_count = len(original_data["archiveresults"])
ok, msg = verify_process_migration(db_path, expected_count)
assert ok, msg
def test_migration_creates_binary_records(migration_08_data):
"""Migration should create and link Binary/NetworkInterface records from migrated Process data."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check Binary records exist
cursor.execute("SELECT COUNT(*) FROM machine_binary")
binary_count = cursor.fetchone()[0]
# Should have at least one binary per unique extractor
extractors = {ar["extractor"] for ar in original_data["archiveresults"]}
assert binary_count >= len(extractors), f"Expected at least {len(extractors)} Binaries, got {binary_count}"
cursor.execute("""
SELECT COUNT(*)
FROM machine_process
WHERE cmd != '[]' AND binary_id IS NULL
""")
missing_binary_count = cursor.fetchone()[0]
assert missing_binary_count == 0
cursor.execute("""
SELECT p.cmd, b.name, b.abspath
FROM machine_process p
JOIN machine_binary b ON p.binary_id = b.id
WHERE p.cmd != '[]'
""")
rows = cursor.fetchall()
assert rows
for cmd_raw, binary_name, binary_abspath in rows:
cmd = json.loads(cmd_raw)
assert binary_name == cmd[0]
assert binary_abspath == cmd[0]
cursor.execute("SELECT COUNT(*) FROM machine_process WHERE iface_id IS NULL")
missing_iface_count = cursor.fetchone()[0]
assert missing_iface_count == 0
conn.close()
def test_migration_preserves_cmd_data(migration_08_data):
"""Migration should preserve cmd data in Process.cmd field."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check that Process records have cmd arrays
cursor.execute("SELECT cmd FROM machine_process WHERE cmd != '[]'")
cmd_records = cursor.fetchall()
# All Processes should have non-empty cmd (test data has json.dumps([extractor, '--version']))
expected_count = len(original_data["archiveresults"])
assert len(cmd_records) == expected_count, f"Expected {expected_count} Processes with cmd, got {len(cmd_records)}"
conn.close()
def test_no_duplicate_snapshots_after_migration(migration_08_data):
"""Migration should not create duplicate snapshots."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
# Check for duplicate URLs
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("""
SELECT url, COUNT(*) as cnt FROM core_snapshot
GROUP BY url HAVING cnt > 1
""")
duplicates = cursor.fetchall()
conn.close()
assert len(duplicates) == 0, f"Found duplicate URLs: {duplicates}"
def test_no_orphaned_archiveresults_after_migration(migration_08_data):
"""Migration should not leave orphaned ArchiveResults."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
ok, msg = verify_foreign_keys(db_path)
assert ok, msg
def test_timestamps_preserved_after_migration(migration_08_data):
"""Migration should preserve original timestamps."""
work_dir, db_path, original_data = migration_08_data
original_timestamps = {s["url"]: s["timestamp"] for s in original_data["snapshots"]}
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT url, timestamp FROM core_snapshot")
migrated_timestamps = {row[0]: row[1] for row in cursor.fetchall()}
conn.close()
for url, original_ts in original_timestamps.items():
assert migrated_timestamps.get(url) == original_ts, f"Timestamp changed for {url}: {original_ts} -> {migrated_timestamps.get(url)}"
def test_crawl_data_preserved_after_migration(migration_08_data):
"""Migration should preserve crawl metadata (urls, label, status)."""
work_dir, db_path, original_data = migration_08_data
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
# Check each crawl's data is preserved
for crawl in original_data["crawls"]:
cursor.execute("SELECT urls, label, status FROM crawls_crawl WHERE id = ?", (crawl["id"],))
row = cursor.fetchone()
assert row is not None, f"Crawl {crawl['id']} not found after migration"
assert row[0] == crawl["urls"], f"URLs mismatch for crawl {crawl['id']}"
assert row[1] == crawl["label"], f"Label mismatch for crawl {crawl['id']}"
assert row[2] == crawl["status"], f"Status mismatch for crawl {crawl['id']}"
conn.close()
def test_tag_associations_preserved_after_migration(migration_08_data):
"""Migration should preserve snapshot-tag associations."""
work_dir, db_path, original_data = migration_08_data
# Count tag associations before migration
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
original_count = cursor.fetchone()[0]
conn.close()
result = run_archivebox_migration_cmd(work_dir, ["init"], timeout=45)
assert result.returncode == 0, f"Init failed: {result.stderr}"
# Count tag associations after migration
conn = sqlite3.connect(str(db_path))
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM core_snapshot_tags")
migrated_count = cursor.fetchone()[0]
conn.close()
assert migrated_count == original_count, f"Tag associations changed: {original_count} -> {migrated_count}"
def test_update_migrates_db_snapshot_when_legacy_index_missing(tmp_path):
"""A legacy folder with no index file should still migrate if its timestamp exists in DB."""
work_dir = tmp_path
-695
View File
@@ -1,695 +0,0 @@
import asyncio
import json
import os
import re
import shutil
import socket
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FutureTimeoutError
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qs, quote, urlsplit
import pytest
import requests
from asgiref.testing import ApplicationCommunicator
from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
from archivebox.config.common import get_config
pytestmark = pytest.mark.django_db(transaction=True)
def _free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return int(sock.getsockname()[1])
def _reset_runtime_config() -> None:
from archivebox.config import common
from archivebox.machine.models import Machine
for value in vars(common).values():
cache_clear = getattr(value, "cache_clear", None)
if cache_clear is not None:
cache_clear()
Machine.current(refresh=True)
def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
os.chdir(data_dir)
result = run_archivebox_cmd(
["config", "--set", *values],
cwd=data_dir,
env=env,
timeout=120,
)
assert result.returncode == 0, result.stderr or result.stdout
_reset_runtime_config()
@pytest.fixture
def opencode_archive_config(initialized_archive):
port = _free_port()
state_dir = initialized_archive / "opencode"
env = os.environ.copy()
env.update(
{
"ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
"OPENCODE_ENABLED": "True",
"OPENCODE_HOST": "127.0.0.1",
"OPENCODE_PORT": str(port),
"OPENCODE_WORKDIR": str(initialized_archive),
"OPENCODE_STATE_DIR": str(state_dir),
"OPENCODE_TIMEOUT": "60",
},
)
_set_archivebox_config(
initialized_archive,
"OPENCODE_ENABLED=True",
"OPENCODE_HOST=127.0.0.1",
f"OPENCODE_PORT={port}",
f"OPENCODE_WORKDIR={initialized_archive}",
f"OPENCODE_STATE_DIR={state_dir}",
"OPENCODE_TIMEOUT=60",
env=env,
)
return SimpleNamespace(data_dir=initialized_archive, port=port, state_dir=state_dir, env=env)
@pytest.fixture
def installed_opencode(opencode_archive_config):
from abx_plugins.plugins.opencode import runtime
install = run_archivebox_cmd(
["install", "opencode", "--binproviders=env,pnpm"],
cwd=opencode_archive_config.data_dir,
env=opencode_archive_config.env,
timeout=1200,
)
assert install.returncode == 0, install.stderr or install.stdout
_reset_runtime_config()
config = get_config().model_dump(mode="json")
settings = runtime._settings(config, opencode_archive_config.data_dir)
settings["archivebox_base_url"] = "http://admin.archivebox.localhost:5797"
settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:5797/admin"
settings["archivebox_api_url"] = "http://admin.archivebox.localhost:5797/api/"
binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"])
version = binary.exec(
cmd=("--version",),
env={**os.environ, **binary_env},
timeout=120,
)
assert version.returncode == 0, version.stderr or version.stdout
return SimpleNamespace(config=opencode_archive_config, settings=settings)
@pytest.fixture
def live_opencode(installed_opencode):
from abx_plugins.plugins.opencode import runtime
settings = installed_opencode.settings
ok, error = runtime._ensure_opencode(settings)
assert ok, error
process = runtime._PROCESS
assert process is not None
try:
yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process)
finally:
runtime._stop_owned_process()
def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
from archivebox.machine.models import Machine
from abx_plugins.plugins.opencode import runtime
os.chdir(initialized_archive)
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
_reset_runtime_config()
assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 404
assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None
def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive):
_set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False")
assert get_config().OPENCODE_ENABLED is False
assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404
for path in ("/add/", "/admin/core/snapshot/"):
response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert b'href="/admin/agent"' not in response.content
def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode):
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 302
assert "/admin/login/" in response.headers["Location"]
next_path = "/admin/agent?x=1&next=https://example.com"
response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 302
assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]}
user = django_user_model.objects.create_user(username="regular", password="testpassword")
client.force_login(user)
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 403
def test_opencode_proxy_blocks_cross_origin_mutation(admin_client, db, live_opencode):
response = admin_client.post(
"/admin/agent/opencode/session",
data=b"{}",
content_type="application/json",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_ORIGIN="https://evil.example",
)
assert response.status_code == 403
def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_opencode):
response = admin_client.post(
"/admin/agent/opencode/session",
data=b"{}",
content_type="application/json",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="cross-site",
)
assert response.status_code == 403
def test_opencode_cold_agent_wrapper_returns_before_server_starts(admin_client, installed_opencode):
import time
from abx_plugins.plugins.opencode import runtime
assert not runtime._owned_process_running()
started = time.monotonic()
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert time.monotonic() - started < 3
assert not runtime._owned_process_running()
assert b'id="opencode-agent-welcome"' in response.content
assert f'<iframe src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
frame_path = runtime._project_route(live_opencode.config.data_dir)
assert response.status_code == 200
assert f'<iframe src="{frame_path}"'.encode() in response.content
assert b'id="opencode-agent-welcome"' in response.content
assert b'id="header"' in response.content
assert b'id="progress-monitor"' in response.content
assert b'<a href="/admin/agent" class="navbar-item navbar-ai">' in response.content
add_page = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
assert add_page.status_code == 200
assert '<a href="/admin/agent">💬 Crawl with AI</a>'.encode() in add_page.content
assert response.context["proxy_prefix"] == runtime._PROXY_PREFIX
assert b"/_archivebox/health" not in response.content
assert b"window.setInterval(check, 3000)" not in response.content
assert response.headers["X-Frame-Options"] == "DENY"
assert response.headers["Content-Security-Policy"] == "frame-ancestors 'none'"
frame = admin_client.get(
frame_path,
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert frame.status_code == 302
assert frame.headers["Location"].startswith(frame_path + "/")
session = admin_client.get(
frame.headers["Location"],
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert session.status_code == 200
assert session.headers["X-Frame-Options"] == "SAMEORIGIN"
assert session.headers["Content-Security-Policy"] == "frame-ancestors 'self'"
def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):
workdir = str(live_opencode.config.data_dir.resolve())
encoded_workdir = quote(workdir)
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert agent.status_code == 200
from abx_plugins.plugins.opencode import runtime
frame_path = runtime._project_route(live_opencode.config.data_dir)
frame = admin_client.get(frame_path, HTTP_HOST=ADMIN_TEST_HOST)
assert frame.status_code == 302
session_id = frame.headers["Location"].rsplit("/", 1)[-1]
project = admin_client.get(
f"/admin/agent/opencode/project/current?directory={encoded_workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert project.status_code == 200
assert project.json()["id"] == "global"
assert not project.json().get("vcs")
path = admin_client.get(
f"/admin/agent/opencode/path?directory={encoded_workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert path.status_code == 200
assert path.json()["directory"] == workdir
sessions = admin_client.get(
f"/admin/agent/opencode/session?directory={encoded_workdir}&roots=true&limit=55",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert sessions.status_code == 200
assert any(session["id"] == session_id and session["directory"] == workdir for session in sessions.json())
assert not (Path(workdir) / ".git").exists()
def test_opencode_proxy_restarts_server_for_an_existing_agent_page(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
old_process = runtime._PROCESS
runtime._stop_owned_process()
response = admin_client.get(
"/admin/agent/opencode/global/health",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
assert runtime._PROCESS is not None
assert runtime._PROCESS is not old_process
assert runtime._PROCESS.poll() is None
def test_concurrent_opencode_startup_waits_until_server_is_ready(live_opencode):
from abx_plugins.plugins.opencode import runtime
runtime._stop_owned_process()
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(runtime._ensure_opencode, [live_opencode.settings] * 2))
assert results == [(True, ""), (True, "")]
assert runtime._health(live_opencode.settings)
def test_opencode_does_not_probe_or_replace_a_ready_owned_process(live_opencode):
from abx_plugins.plugins.opencode import runtime
process = runtime._PROCESS
settings = {**live_opencode.settings, "port": _free_port()}
settings["origin"] = f"http://{settings['host']}:{settings['port']}"
ok, error = runtime._ensure_opencode(settings)
assert ok, error
assert process is not None
assert runtime._PROCESS is process
assert process.poll() is None
def test_opencode_proxy_does_not_wait_for_recovery_lock(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
workdir = quote(str(live_opencode.config.data_dir.resolve()))
assert runtime._owned_process_ready()
executor = ThreadPoolExecutor(max_workers=1)
runtime._PROCESS_LOCK.acquire()
try:
request = executor.submit(
admin_client.get,
f"/admin/agent/opencode/path?directory={workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
response = request.result(timeout=5)
finally:
runtime._PROCESS_LOCK.release()
executor.shutdown(wait=True)
assert response.status_code == 200
assert str(live_opencode.config.data_dir.resolve()).encode() in response.content
def test_opencode_proxy_waits_for_owned_process_readiness(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
process = runtime._PROCESS
assert process is not None
runtime._PROCESS_READY = None
workdir = quote(str(live_opencode.config.data_dir.resolve()))
response = admin_client.get(
f"/admin/agent/opencode/path?directory={workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
assert runtime._PROCESS is process
assert runtime._PROCESS_READY is process
def test_opencode_proxy_preserves_protocol_headers(admin_client, live_opencode):
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
created = admin_client.post(
"/admin/agent/opencode/pty",
data={"command": "/bin/sh", "args": []},
content_type="application/json",
**headers,
)
assert created.status_code == 200
path = f"/admin/agent/opencode/pty/{created.json()['id']}"
try:
response = admin_client.post(path + "/connect-token", HTTP_X_OPENCODE_TICKET="1", **headers)
assert response.status_code == 200, response.content
assert response.json()["ticket"]
finally:
deleted = admin_client.delete(path, **headers)
assert deleted.status_code == 200, deleted.content
def test_opencode_oauth_callback_waits_for_user_and_preserves_cancellation(live_opencode):
from abx_plugins.plugins.opencode import runtime
settings = {**live_opencode.settings, "timeout": 1}
headers = {"Content-Type": "application/json"}
status, _, body = runtime.proxy(settings, "POST", "provider/openai/oauth/authorize", (), headers, b'{"method":0}')
assert status == 200
authorization = json.loads(body)
assert urlsplit(authorization["url"]).hostname == "auth.openai.com"
redirect = parse_qs(urlsplit(authorization["url"]).query)["redirect_uri"][0]
callback_origin = urlsplit(redirect)
assert callback_origin.hostname == "localhost"
with ThreadPoolExecutor(max_workers=1) as executor:
callback = executor.submit(runtime.proxy, settings, "POST", "provider/openai/oauth/callback", (), headers, b'{"method":0}')
try:
# A real pending authorization must outlive the ordinary API read
# timeout. No tokens or substituted provider responses are used.
with pytest.raises(FutureTimeoutError):
callback.result(timeout=2)
finally:
cancelled = requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=5)
assert cancelled.status_code == 200
assert cancelled.text == "Login cancelled"
status, response_headers, body = callback.result(timeout=5)
assert status == 500
assert response_headers["Content-Type"].startswith("application/json")
error = json.loads(body)
assert error["name"] == "UnknownError"
assert error["data"]["ref"].startswith("err_")
runtime._stop_owned_process()
# The pnpm launcher can exit before its server child. Stopping the owned
# process must release both listeners so another collection can authorize.
with pytest.raises(requests.ConnectionError):
requests.get(settings["origin"] + "/global/health", timeout=2)
with pytest.raises(requests.ConnectionError):
requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=2)
def test_opencode_static_assets_cache_privately_and_revalidate(admin_client, live_opencode):
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
page = admin_client.get("/admin/agent/opencode/", **headers)
assert page.status_code == 200
asset = re.search(rb'src="(/admin/agent/opencode/assets/[^\"]+\.js)"', page.content)
assert asset is not None
path = asset[1].decode()
response = admin_client.get(path, **headers)
assert response.status_code == 200
assert "private" in response.headers["Cache-Control"]
assert "max-age=" in response.headers["Cache-Control"]
assert b"/admin/agent/opencode" in response.content
repeated = admin_client.get(path, HTTP_IF_NONE_MATCH=response.headers["ETag"], **headers)
assert repeated.status_code == 304
assert repeated.content == b""
assert repeated.headers["ETag"] == response.headers["ETag"]
assert admin_client.get("/admin/agent/opencode/path", **headers).headers["Cache-Control"] == "no-store"
def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
response = admin_client.get(
"/admin/agent/opencode/global/event",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
assert response.streaming
assert response.is_async
assert response.headers["X-Accel-Buffering"] == "no"
assert response.headers["Cache-Control"] == "no-store"
@pytest.mark.parametrize("path", ["event", "global/event"])
def test_opencode_proxy_sse_delivers_first_event_immediately(admin_client, live_opencode, path):
from abx_plugins.plugins.opencode import runtime
# The real /event endpoint emits heartbeats indefinitely. A bounded
# upstream read exposes accidental buffering without hanging the test.
status, headers, body = runtime.proxy(
{**live_opencode.settings, "timeout": 1},
"GET",
path,
(),
{},
b"",
)
assert status == 200
assert not isinstance(body, bytes)
assert headers["Content-Type"] == "text/event-stream"
asyncio.run(body.aclose())
response = admin_client.get(
f"/admin/agent/opencode/{path}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
async def first_event():
stream = response.streaming_content
try:
async with asyncio.timeout(2):
return await anext(stream)
finally:
await stream.aclose()
assert b"server.connected" in asyncio.run(first_event())
def test_opencode_proxy_sse_returns_headers_before_restart_finishes(admin_client, live_opencode):
from archivebox.core.asgi import application
from abx_plugins.plugins.opencode import runtime
from django.conf import settings as django_settings
owned_process = runtime._PROCESS
assert owned_process is not None
session_cookie_name = django_settings.SESSION_COOKIE_NAME
session_cookie = admin_client.cookies[session_cookie_name].value
path = "/admin/agent/opencode/global/event"
async def request_event_stream():
communicator = ApplicationCommunicator(
application,
{
"type": "http",
"asgi": {"version": "3.0"},
"http_version": "1.1",
"method": "GET",
"scheme": "http",
"path": path,
"raw_path": path.encode(),
"query_string": b"",
"headers": [
(b"host", ADMIN_TEST_HOST.encode()),
(b"cookie", f"{session_cookie_name}={session_cookie}".encode()),
(b"sec-fetch-site", b"same-origin"),
],
"client": ("127.0.0.1", 12345),
"server": ("127.0.0.1", 5797),
},
)
runtime._PROCESS_LOCK.acquire()
runtime._PROCESS = None
try:
await communicator.send_input({"type": "http.request", "body": b"", "more_body": False})
response_start = await communicator.receive_output(timeout=2)
assert response_start["type"] == "http.response.start"
assert response_start["status"] == 200
finally:
try:
await communicator.send_input({"type": "http.disconnect"})
await communicator.wait(timeout=5)
finally:
runtime._PROCESS = owned_process
runtime._PROCESS_LOCK.release()
await asyncio.get_running_loop().shutdown_default_executor()
asyncio.run(request_event_stream())
assert runtime._PROCESS is owned_process
assert owned_process.poll() is None
def test_opencode_starts_with_isolated_state(admin_client, live_opencode):
workdir = str(live_opencode.config.data_dir.resolve())
state_dir = live_opencode.config.state_dir
assert not (Path(workdir) / ".git").exists()
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert agent.status_code == 200
from abx_plugins.plugins.opencode import runtime
frame = admin_client.get(
runtime._project_route(live_opencode.config.data_dir),
HTTP_HOST=ADMIN_TEST_HOST,
)
assert frame.status_code == 302
assert not (Path(workdir) / ".git").exists()
project = requests.get(
f"{live_opencode.settings['origin']}/project/current",
params={"directory": workdir},
timeout=live_opencode.settings["timeout"],
)
project.raise_for_status()
config = requests.get(
f"{live_opencode.settings['origin']}/global/config",
timeout=live_opencode.settings["timeout"],
)
config.raise_for_status()
assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
assert project.json()["id"] == "global"
assert not project.json().get("vcs")
assert not config.json().get("model")
assert config.json()["snapshot"] is False
assert live_opencode.process.poll() is None
path = requests.get(
f"{live_opencode.settings['origin']}/path",
params={"directory": workdir},
timeout=live_opencode.settings["timeout"],
)
path.raise_for_status()
assert Path(path.json()["directory"]).resolve() == Path(workdir)
diff = requests.get(
f"{live_opencode.settings['origin']}/vcs/diff",
params={"directory": workdir, "mode": "git"},
timeout=5,
)
diff.raise_for_status()
assert diff.json() == []
assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
assert (state_dir / "SKILL.md").is_file()
assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
def test_opencode_invalid_state_does_not_break_archivebox(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
runtime._stop_owned_process()
invalid_state = live_opencode.config.state_dir / "config"
invalid_state.rename(live_opencode.config.state_dir / "saved-config")
invalid_state.write_text("Preserve this file.")
wrapper = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert wrapper.status_code == 200
for url in (
runtime._project_route(live_opencode.config.data_dir),
"/admin/agent/opencode/global/health",
):
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 503
assert str(invalid_state).encode() not in response.content
stream = admin_client.get("/admin/agent/opencode/global/event", HTTP_HOST=ADMIN_TEST_HOST)
assert stream.status_code == 200
async def read_failure():
return b"".join([chunk async for chunk in stream.streaming_content])
assert asyncio.run(read_failure()) == b'event: error\ndata: {"error":"OpenCode unavailable"}\n\n'
for url in ("/health/", "/add/", "/admin/core/snapshot/"):
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert invalid_state.read_text() == "Preserve this file."
@pytest.mark.parametrize(
("damaged_file", "missing"),
[
("runtime.py", True),
("templates/agent.html", True),
("templates/agent.html", False),
("templates/navigation.html", False),
("templates/add.html", False),
],
)
def test_opencode_incomplete_install_does_not_break_archivebox(installed_opencode, tmp_path, damaged_file, missing):
import abx_plugins
# Exercise a genuinely incomplete installation in a separate process;
# never alter the shared package or intercept Python imports.
site = tmp_path / "site"
installed = site / "abx_plugins"
shutil.copytree(Path(abx_plugins.__file__).parent, installed, ignore=shutil.ignore_patterns("__pycache__", "tests"))
damaged_path = installed / "plugins" / "opencode" / damaged_file
if missing:
damaged_path.unlink()
else:
damaged_path.write_text("{% invalid_template_tag %}")
expected_status = 200 if damaged_file in {"templates/navigation.html", "templates/add.html"} else 503
script = f"""
import sys
from pathlib import Path
import abx_plugins
from django.test import Client
from django.contrib.auth import get_user_model
assert Path(abx_plugins.__file__).is_relative_to({str(site)!r})
user = get_user_model().objects.create_superuser(username='optional-service-test')
client = Client(HTTP_HOST={ADMIN_TEST_HOST!r})
client.force_login(user)
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
assert client.get(path).status_code == 200, path
assert 'abx_plugins.plugins.opencode.runtime' not in sys.modules
response = client.get('/admin/agent')
assert response.status_code == {expected_status}, response.status_code
if response.status_code == 503:
assert response.content == b'AI service unavailable. See server logs.'
if {damaged_file != "runtime.py"!r}:
from abx_plugins.plugins.opencode import runtime
assert runtime._PROCESS is None
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
assert client.get(path).status_code == 200, path
print('OPTIONAL_SERVICE_FAILURE_ISOLATED')
"""
result = run_archivebox_cmd(
["shell", "-c", script],
cwd=installed_opencode.config.data_dir,
env={**installed_opencode.config.env, "PYTHONPATH": str(site)},
timeout=90,
)
assert result.returncode == 0, result.stderr or result.stdout
assert "OPTIONAL_SERVICE_FAILURE_ISOLATED" in result.stdout
+291
View File
@@ -0,0 +1,291 @@
import os
import shutil
import socket
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import quote
import pytest
import requests
from archivebox.tests.conftest import ADMIN_TEST_HOST, run_archivebox_cmd
from archivebox.config.common import get_config
pytestmark = pytest.mark.django_db(transaction=True)
def _free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return int(sock.getsockname()[1])
def _reset_runtime_config() -> None:
from archivebox.config import common
from archivebox.machine.models import Machine
for value in vars(common).values():
cache_clear = getattr(value, "cache_clear", None)
if cache_clear is not None:
cache_clear()
Machine.current()
def _set_archivebox_config(data_dir: Path, *values: str, env: dict[str, str] | None = None) -> None:
os.chdir(data_dir)
result = run_archivebox_cmd(
["config", "--set", *values],
cwd=data_dir,
env=env,
timeout=120,
)
assert result.returncode == 0, result.stderr or result.stdout
_reset_runtime_config()
@pytest.fixture
def opencode_archive_config(initialized_archive):
port = _free_port()
state_dir = initialized_archive / "opencode"
env = os.environ.copy()
env.update(
{
"ARCHIVEBOX_ALLOW_NO_UNIX_SOCKETS": "true",
"OPENCODE_ENABLED": "True",
"OPENCODE_HOST": "127.0.0.1",
"OPENCODE_PORT": str(port),
"OPENCODE_WORKDIR": str(initialized_archive),
"OPENCODE_STATE_DIR": str(state_dir),
"OPENCODE_TIMEOUT": "60",
},
)
_set_archivebox_config(
initialized_archive,
"OPENCODE_ENABLED=True",
"OPENCODE_HOST=127.0.0.1",
f"OPENCODE_PORT={port}",
f"OPENCODE_WORKDIR={initialized_archive}",
f"OPENCODE_STATE_DIR={state_dir}",
"OPENCODE_TIMEOUT=60",
env=env,
)
return SimpleNamespace(data_dir=initialized_archive, port=port, state_dir=state_dir, env=env)
@pytest.fixture
def installed_opencode(opencode_archive_config):
from abx_plugins.plugins.opencode import runtime
install = run_archivebox_cmd(
["install", "opencode", "--binproviders=env,pnpm"],
cwd=opencode_archive_config.data_dir,
env=opencode_archive_config.env,
timeout=1200,
)
assert install.returncode == 0, install.stderr or install.stdout
_reset_runtime_config()
config = get_config().model_dump(mode="json")
settings = runtime._settings(config, opencode_archive_config.data_dir)
settings["archivebox_base_url"] = "http://admin.archivebox.localhost:5797"
settings["archivebox_admin_url"] = "http://admin.archivebox.localhost:5797/admin"
settings["archivebox_api_url"] = "http://admin.archivebox.localhost:5797/api/"
binary, binary_env = runtime._resolve_binary(settings["binary"], settings["config"])
version = binary.exec(
cmd=("--version",),
env={**os.environ, **binary_env},
timeout=120,
)
assert version.returncode == 0, version.stderr or version.stdout
return SimpleNamespace(config=opencode_archive_config, settings=settings)
@pytest.fixture
def live_opencode(installed_opencode):
from abx_plugins.plugins.opencode import runtime
settings = installed_opencode.settings
ok, error = runtime._ensure_opencode(settings)
assert ok, error
process = runtime._PROCESS
assert process is not None
try:
yield SimpleNamespace(config=installed_opencode.config, settings=settings, process=process)
finally:
runtime._stop_owned_process()
def test_opencode_disabled_via_cli_stays_disabled(admin_client, initialized_archive):
_set_archivebox_config(initialized_archive, "OPENCODE_ENABLED=False")
assert get_config().OPENCODE_ENABLED is False
assert admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST).status_code == 404
for path in ("/add/", "/admin/core/snapshot/"):
response = admin_client.get(path, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert b'href="/admin/agent"' not in response.content
def test_opencode_proxy_blocks_cross_site_fetch_metadata(admin_client, db, live_opencode):
response = admin_client.post(
"/admin/agent/opencode/session",
data=b"{}",
content_type="application/json",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="cross-site",
)
assert response.status_code == 403
def test_opencode_proxy_waits_for_owned_process_readiness(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
process = runtime._PROCESS
assert process is not None
runtime._PROCESS_READY = None
workdir = quote(str(live_opencode.config.data_dir.resolve()))
response = admin_client.get(
f"/admin/agent/opencode/path?directory={workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
assert runtime._PROCESS is process
assert runtime._PROCESS_READY is process
def test_opencode_proxy_preserves_protocol_headers(admin_client, live_opencode):
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
created = admin_client.post(
"/admin/agent/opencode/pty",
data={"command": "/bin/sh", "args": []},
content_type="application/json",
**headers,
)
assert created.status_code == 200
path = f"/admin/agent/opencode/pty/{created.json()['id']}"
try:
response = admin_client.post(path + "/connect-token", HTTP_X_OPENCODE_TICKET="1", **headers)
assert response.status_code == 200, response.content
assert response.json()["ticket"]
finally:
deleted = admin_client.delete(path, **headers)
assert deleted.status_code == 200, deleted.content
def test_opencode_starts_with_isolated_state(admin_client, live_opencode):
workdir = str(live_opencode.config.data_dir.resolve())
state_dir = live_opencode.config.state_dir
assert not (Path(workdir) / ".git").exists()
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert agent.status_code == 200
from abx_plugins.plugins.opencode import runtime
frame = admin_client.get(
runtime._project_route(live_opencode.config.data_dir),
HTTP_HOST=ADMIN_TEST_HOST,
)
assert frame.status_code == 302
assert not (Path(workdir) / ".git").exists()
project = requests.get(
f"{live_opencode.settings['origin']}/project/current",
params={"directory": workdir},
timeout=live_opencode.settings["timeout"],
)
project.raise_for_status()
config = requests.get(
f"{live_opencode.settings['origin']}/global/config",
timeout=live_opencode.settings["timeout"],
)
config.raise_for_status()
assert Path(live_opencode.settings["workdir"]).resolve() == Path(workdir)
assert project.json()["id"] == "global"
assert not project.json().get("vcs")
assert not config.json().get("model")
assert config.json()["snapshot"] is False
assert live_opencode.process.poll() is None
path = requests.get(
f"{live_opencode.settings['origin']}/path",
params={"directory": workdir},
timeout=live_opencode.settings["timeout"],
)
path.raise_for_status()
assert Path(path.json()["directory"]).resolve() == Path(workdir)
diff = requests.get(
f"{live_opencode.settings['origin']}/vcs/diff",
params={"directory": workdir, "mode": "git"},
timeout=5,
)
diff.raise_for_status()
assert diff.json() == []
assert (state_dir / "data" / "opencode" / "opencode.db").is_file()
assert (state_dir / "SKILL.md").is_file()
assert (state_dir / "config" / "opencode" / "skills" / "archivebox" / "SKILL.md").resolve() == state_dir / "SKILL.md"
@pytest.mark.parametrize(
("damaged_file", "missing"),
[
("runtime.py", True),
("templates/agent.html", True),
("templates/agent.html", False),
("templates/navigation.html", False),
("templates/add.html", False),
],
)
def test_opencode_incomplete_install_does_not_break_archivebox(installed_opencode, tmp_path, damaged_file, missing):
import abx_plugins
# Exercise a genuinely incomplete installation in a separate process;
# never alter the shared package or intercept Python imports.
site = tmp_path / "site"
installed = site / "abx_plugins"
shutil.copytree(Path(abx_plugins.__file__).parent, installed, ignore=shutil.ignore_patterns("__pycache__", "tests"))
damaged_path = installed / "plugins" / "opencode" / damaged_file
if missing:
damaged_path.unlink()
else:
damaged_path.write_text("{% invalid_template_tag %}")
expected_status = 200 if damaged_file in {"templates/navigation.html", "templates/add.html"} else 503
script = f"""
import sys
from pathlib import Path
import abx_plugins
from django.test import Client
from django.contrib.auth import get_user_model
assert Path(abx_plugins.__file__).is_relative_to({str(site)!r})
user = get_user_model().objects.create_superuser(username='optional-service-test')
client = Client(HTTP_HOST={ADMIN_TEST_HOST!r})
client.force_login(user)
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
assert client.get(path).status_code == 200, path
assert 'abx_plugins.plugins.opencode.runtime' not in sys.modules
response = client.get('/admin/agent')
assert response.status_code == {expected_status}, response.status_code
if response.status_code == 503:
assert response.content == b'AI service unavailable. See server logs.'
if {damaged_file != "runtime.py"!r}:
from abx_plugins.plugins.opencode import runtime
assert runtime._PROCESS is None
for path in ('/health/', '/add/', '/admin/core/snapshot/'):
assert client.get(path).status_code == 200, path
print('OPTIONAL_SERVICE_FAILURE_ISOLATED')
"""
result = run_archivebox_cmd(
["shell", "-c", script],
cwd=installed_opencode.config.data_dir,
env={**installed_opencode.config.env, "PYTHONPATH": str(site)},
timeout=90,
)
assert result.returncode == 0, result.stderr or result.stdout
assert "OPTIONAL_SERVICE_FAILURE_ISOLATED" in result.stdout
+251
View File
@@ -0,0 +1,251 @@
import asyncio
import json
import os
import re
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FutureTimeoutError
from urllib.parse import parse_qs, urlsplit
import pytest
import requests
from asgiref.testing import ApplicationCommunicator
from archivebox.tests.conftest import ADMIN_TEST_HOST
from archivebox.config.common import get_config
from .test_opencode_agent_1 import (
pytestmark as pytestmark,
_free_port as _free_port,
_reset_runtime_config as _reset_runtime_config,
_set_archivebox_config as _set_archivebox_config,
opencode_archive_config as opencode_archive_config,
installed_opencode as installed_opencode,
live_opencode as live_opencode,
)
def test_opencode_disabled_route_does_not_start_server(client, initialized_archive):
from archivebox.machine.models import Machine
from abx_plugins.plugins.opencode import runtime
os.chdir(initialized_archive)
Machine.from_json({"config": {"OPENCODE_ENABLED": False}})
_reset_runtime_config()
assert get_config().model_dump(mode="json")["OPENCODE_ENABLED"] is False
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 404
assert runtime._PROCESS is None or runtime._PROCESS.poll() is not None
def test_opencode_agent_requires_superuser_when_enabled(client, db, django_user_model, live_opencode):
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 302
assert "/admin/login/" in response.headers["Location"]
next_path = "/admin/agent?x=1&next=https://example.com"
response = client.get(next_path, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 302
assert parse_qs(urlsplit(response.headers["Location"]).query) == {"next": [next_path]}
user = django_user_model.objects.create_user(username="regular", password="testpassword")
client.force_login(user)
response = client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 403
def test_opencode_agent_superuser_gets_admin_wrapper(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
frame_path = runtime._project_route(live_opencode.config.data_dir)
assert response.status_code == 200
assert f'<iframe src="{frame_path}"'.encode() in response.content
assert b'id="opencode-agent-welcome"' in response.content
assert b'id="header"' in response.content
assert b'id="progress-monitor"' in response.content
assert b'<a href="/admin/agent" class="navbar-item navbar-ai">' in response.content
add_page = admin_client.get("/add/", HTTP_HOST=ADMIN_TEST_HOST)
assert add_page.status_code == 200
assert '<a href="/admin/agent">💬 Crawl with AI</a>'.encode() in add_page.content
assert response.context["proxy_prefix"] == runtime._PROXY_PREFIX
assert b"/_archivebox/health" not in response.content
assert b"window.setInterval(check, 3000)" not in response.content
assert response.headers["X-Frame-Options"] == "DENY"
assert response.headers["Content-Security-Policy"] == "frame-ancestors 'none'"
frame = admin_client.get(
frame_path,
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert frame.status_code == 302
assert frame.headers["Location"].startswith(frame_path + "/")
session = admin_client.get(
frame.headers["Location"],
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert session.status_code == 200
assert session.headers["X-Frame-Options"] == "SAMEORIGIN"
assert session.headers["Content-Security-Policy"] == "frame-ancestors 'self'"
def test_opencode_oauth_callback_waits_for_user_and_preserves_cancellation(live_opencode):
from abx_plugins.plugins.opencode import runtime
settings = {**live_opencode.settings, "timeout": 1}
headers = {"Content-Type": "application/json"}
status, _, body = runtime.proxy(settings, "POST", "provider/openai/oauth/authorize", (), headers, b'{"method":0}')
assert status == 200
authorization = json.loads(body)
assert urlsplit(authorization["url"]).hostname == "auth.openai.com"
redirect = parse_qs(urlsplit(authorization["url"]).query)["redirect_uri"][0]
callback_origin = urlsplit(redirect)
assert callback_origin.hostname == "localhost"
with ThreadPoolExecutor(max_workers=1) as executor:
callback = executor.submit(runtime.proxy, settings, "POST", "provider/openai/oauth/callback", (), headers, b'{"method":0}')
try:
# A real pending authorization must outlive the ordinary API read
# timeout. No tokens or substituted provider responses are used.
with pytest.raises(FutureTimeoutError):
callback.result(timeout=2)
finally:
cancelled = requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=5)
assert cancelled.status_code == 200
assert cancelled.text == "Login cancelled"
status, response_headers, body = callback.result(timeout=5)
assert status == 500
assert response_headers["Content-Type"].startswith("application/json")
error = json.loads(body)
assert error["name"] == "UnknownError"
assert error["data"]["ref"].startswith("err_")
runtime._stop_owned_process()
# The pnpm launcher can exit before its server child. Stopping the owned
# process must release both listeners so another collection can authorize.
with pytest.raises(requests.ConnectionError):
requests.get(settings["origin"] + "/global/health", timeout=2)
with pytest.raises(requests.ConnectionError):
requests.get(f"{callback_origin.scheme}://{callback_origin.netloc}/cancel", timeout=2)
def test_opencode_static_assets_cache_privately_and_revalidate(admin_client, live_opencode):
headers = {"HTTP_HOST": ADMIN_TEST_HOST, "HTTP_SEC_FETCH_SITE": "same-origin"}
page = admin_client.get("/admin/agent/opencode/", **headers)
assert page.status_code == 200
asset = re.search(rb'src="(/admin/agent/opencode/assets/[^\"]+\.js)"', page.content)
assert asset is not None
path = asset[1].decode()
response = admin_client.get(path, **headers)
assert response.status_code == 200
assert "private" in response.headers["Cache-Control"]
assert "max-age=" in response.headers["Cache-Control"]
assert b"/admin/agent/opencode" in response.content
repeated = admin_client.get(path, HTTP_IF_NONE_MATCH=response.headers["ETag"], **headers)
assert repeated.status_code == 304
assert repeated.content == b""
assert repeated.headers["ETag"] == response.headers["ETag"]
assert admin_client.get("/admin/agent/opencode/path", **headers).headers["Cache-Control"] == "no-store"
def test_opencode_proxy_sse_response_is_unbuffered(admin_client, live_opencode):
response = admin_client.get(
"/admin/agent/opencode/global/event",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
assert response.streaming
assert response.is_async
assert response.headers["X-Accel-Buffering"] == "no"
assert response.headers["Cache-Control"] == "no-store"
def test_opencode_proxy_sse_returns_headers_before_restart_finishes(admin_client, live_opencode):
from archivebox.core.asgi import application
from abx_plugins.plugins.opencode import runtime
from django.conf import settings as django_settings
owned_process = runtime._PROCESS
assert owned_process is not None
session_cookie_name = django_settings.SESSION_COOKIE_NAME
session_cookie = admin_client.cookies[session_cookie_name].value
path = "/admin/agent/opencode/global/event"
async def request_event_stream():
communicator = ApplicationCommunicator(
application,
{
"type": "http",
"asgi": {"version": "3.0"},
"http_version": "1.1",
"method": "GET",
"scheme": "http",
"path": path,
"raw_path": path.encode(),
"query_string": b"",
"headers": [
(b"host", ADMIN_TEST_HOST.encode()),
(b"cookie", f"{session_cookie_name}={session_cookie}".encode()),
(b"sec-fetch-site", b"same-origin"),
],
"client": ("127.0.0.1", 12345),
"server": ("127.0.0.1", 5797),
},
)
runtime._PROCESS_LOCK.acquire()
runtime._PROCESS = None
try:
await communicator.send_input({"type": "http.request", "body": b"", "more_body": False})
response_start = await communicator.receive_output(timeout=2)
assert response_start["type"] == "http.response.start"
assert response_start["status"] == 200
finally:
try:
await communicator.send_input({"type": "http.disconnect"})
await communicator.wait(timeout=5)
finally:
runtime._PROCESS = owned_process
runtime._PROCESS_LOCK.release()
await asyncio.get_running_loop().shutdown_default_executor()
asyncio.run(request_event_stream())
assert runtime._PROCESS is owned_process
assert owned_process.poll() is None
def test_opencode_invalid_state_does_not_break_archivebox(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
runtime._stop_owned_process()
invalid_state = live_opencode.config.state_dir / "config"
invalid_state.rename(live_opencode.config.state_dir / "saved-config")
invalid_state.write_text("Preserve this file.")
wrapper = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert wrapper.status_code == 200
for url in (
runtime._project_route(live_opencode.config.data_dir),
"/admin/agent/opencode/global/health",
):
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 503
assert str(invalid_state).encode() not in response.content
stream = admin_client.get("/admin/agent/opencode/global/event", HTTP_HOST=ADMIN_TEST_HOST)
assert stream.status_code == 200
async def read_failure():
return b"".join([chunk async for chunk in stream.streaming_content])
assert asyncio.run(read_failure()) == b'event: error\ndata: {"error":"OpenCode unavailable"}\n\n'
for url in ("/health/", "/add/", "/admin/core/snapshot/"):
response = admin_client.get(url, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert invalid_state.read_text() == "Preserve this file."
+191
View File
@@ -0,0 +1,191 @@
import asyncio
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import quote
import pytest
from archivebox.tests.conftest import ADMIN_TEST_HOST
from .test_opencode_agent_1 import (
pytestmark as pytestmark,
_free_port as _free_port,
_reset_runtime_config as _reset_runtime_config,
_set_archivebox_config as _set_archivebox_config,
opencode_archive_config as opencode_archive_config,
installed_opencode as installed_opencode,
live_opencode as live_opencode,
)
def test_opencode_proxy_blocks_cross_origin_mutation(admin_client, db, live_opencode):
response = admin_client.post(
"/admin/agent/opencode/session",
data=b"{}",
content_type="application/json",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_ORIGIN="https://evil.example",
)
assert response.status_code == 403
def test_opencode_cold_agent_wrapper_returns_before_server_starts(admin_client, installed_opencode):
import time
from abx_plugins.plugins.opencode import runtime
assert not runtime._owned_process_running()
started = time.monotonic()
response = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200
assert time.monotonic() - started < 3
assert not runtime._owned_process_running()
assert b'id="opencode-agent-welcome"' in response.content
assert f'<iframe src="{runtime._project_route(installed_opencode.config.data_dir)}"'.encode() in response.content
def test_opencode_proxy_serves_real_project_and_session(admin_client, live_opencode):
workdir = str(live_opencode.config.data_dir.resolve())
encoded_workdir = quote(workdir)
agent = admin_client.get("/admin/agent", HTTP_HOST=ADMIN_TEST_HOST)
assert agent.status_code == 200
from abx_plugins.plugins.opencode import runtime
frame_path = runtime._project_route(live_opencode.config.data_dir)
frame = admin_client.get(frame_path, HTTP_HOST=ADMIN_TEST_HOST)
assert frame.status_code == 302
session_id = frame.headers["Location"].rsplit("/", 1)[-1]
project = admin_client.get(
f"/admin/agent/opencode/project/current?directory={encoded_workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert project.status_code == 200
assert project.json()["id"] == "global"
assert not project.json().get("vcs")
path = admin_client.get(
f"/admin/agent/opencode/path?directory={encoded_workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert path.status_code == 200
assert path.json()["directory"] == workdir
sessions = admin_client.get(
f"/admin/agent/opencode/session?directory={encoded_workdir}&roots=true&limit=55",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert sessions.status_code == 200
assert any(session["id"] == session_id and session["directory"] == workdir for session in sessions.json())
assert not (Path(workdir) / ".git").exists()
def test_opencode_proxy_restarts_server_for_an_existing_agent_page(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
old_process = runtime._PROCESS
runtime._stop_owned_process()
response = admin_client.get(
"/admin/agent/opencode/global/health",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
assert runtime._PROCESS is not None
assert runtime._PROCESS is not old_process
assert runtime._PROCESS.poll() is None
def test_concurrent_opencode_startup_waits_until_server_is_ready(live_opencode):
from abx_plugins.plugins.opencode import runtime
runtime._stop_owned_process()
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(runtime._ensure_opencode, [live_opencode.settings] * 2))
assert results == [(True, ""), (True, "")]
assert runtime._health(live_opencode.settings)
def test_opencode_does_not_probe_or_replace_a_ready_owned_process(live_opencode):
from abx_plugins.plugins.opencode import runtime
process = runtime._PROCESS
settings = {**live_opencode.settings, "port": _free_port()}
settings["origin"] = f"http://{settings['host']}:{settings['port']}"
ok, error = runtime._ensure_opencode(settings)
assert ok, error
assert process is not None
assert runtime._PROCESS is process
assert process.poll() is None
def test_opencode_proxy_does_not_wait_for_recovery_lock(admin_client, live_opencode):
from abx_plugins.plugins.opencode import runtime
workdir = quote(str(live_opencode.config.data_dir.resolve()))
assert runtime._owned_process_ready()
executor = ThreadPoolExecutor(max_workers=1)
runtime._PROCESS_LOCK.acquire()
try:
request = executor.submit(
admin_client.get,
f"/admin/agent/opencode/path?directory={workdir}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
response = request.result(timeout=5)
finally:
runtime._PROCESS_LOCK.release()
executor.shutdown(wait=True)
assert response.status_code == 200
assert str(live_opencode.config.data_dir.resolve()).encode() in response.content
@pytest.mark.parametrize("path", ["event", "global/event"])
def test_opencode_proxy_sse_delivers_first_event_immediately(admin_client, live_opencode, path):
from abx_plugins.plugins.opencode import runtime
# The real /event endpoint emits heartbeats indefinitely. A bounded
# upstream read exposes accidental buffering without hanging the test.
status, headers, body = runtime.proxy(
{**live_opencode.settings, "timeout": 1},
"GET",
path,
(),
{},
b"",
)
assert status == 200
assert not isinstance(body, bytes)
assert headers["Content-Type"] == "text/event-stream"
asyncio.run(body.aclose())
response = admin_client.get(
f"/admin/agent/opencode/{path}",
HTTP_HOST=ADMIN_TEST_HOST,
HTTP_SEC_FETCH_SITE="same-origin",
)
assert response.status_code == 200
async def first_event():
stream = response.streaming_content
try:
async with asyncio.timeout(2):
return await anext(stream)
finally:
await stream.aclose()
assert b"server.connected" in asyncio.run(first_event())
@@ -1,6 +1,5 @@
"""Exercise the real agent wrapper with Chromium's native storage failures."""
import asyncio
import json
import os
import re
@@ -10,16 +9,64 @@ import pytest
import requests
from .conftest import get_free_port, run_archivebox_cmd, start_archivebox_server, stop_archivebox_process
from .test_opencode_agent import _set_archivebox_config
from .test_opencode_agent import installed_opencode as installed_opencode
from .test_opencode_agent import live_opencode as live_opencode
from .test_opencode_agent import opencode_archive_config as opencode_archive_config
from .test_opencode_agent_1 import _set_archivebox_config
from .test_opencode_agent_1 import installed_opencode as installed_opencode
from .test_opencode_agent_1 import live_opencode as live_opencode
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
from .test_server_security_browser import browser_runtime as browser_runtime
pytestmark = pytest.mark.django_db(transaction=True)
@pytest.fixture
def agent_server(installed_opencode, browser_runtime, request):
port = get_free_port()
mode, subdomains = getattr(request, "param", ("safe-onedomain-nojsreplay", False))
base_url = f"http://archivebox.localhost:{port}" if subdomains else f"http://localhost:{port}"
url = f"http://admin.archivebox.localhost:{port}" if subdomains else base_url
config = installed_opencode.config
_set_archivebox_config(
config.data_dir,
f"BASE_URL={base_url}",
f"SERVER_SECURITY_MODE={mode}",
)
user = run_archivebox_cmd(
[
"shell",
"-c",
"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(username='agent-browser-test', password='test-password'); User.objects.create_user(username='agent-regular-test', password='test-password', is_staff=True)",
],
cwd=config.data_dir,
env=config.env,
)
assert user.returncode == 0, user.stderr or user.stdout
process = start_archivebox_server(config.data_dir, port=port, env=config.env, log_name="agent-browser-server.log")
try:
yield url, config.data_dir, process
finally:
if process.poll() is None:
stop_archivebox_process(process)
def _login_cookie(server_url, username="agent-browser-test"):
session = requests.Session()
login_url = server_url + "/admin/login/"
page = session.get(login_url, timeout=10)
assert page.status_code == 200
token = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', page.text)
assert token is not None
response = session.post(
login_url,
data={"username": username, "password": "test-password", "csrfmiddlewaretoken": token[1]},
headers={"Referer": login_url},
allow_redirects=False,
timeout=10,
)
assert response.status_code == 302
return "; ".join(f"{key}={value}" for key, value in session.cookies.items())
@pytest.mark.parametrize(
"agent_server",
[
@@ -231,262 +278,3 @@ const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
)
assert result.returncode == 0, result.stderr or result.stdout
assert "AGENT_NAVIGATION_OK" in result.stdout
@pytest.fixture
def agent_server(installed_opencode, browser_runtime, request):
port = get_free_port()
mode, subdomains = getattr(request, "param", ("safe-onedomain-nojsreplay", False))
base_url = f"http://archivebox.localhost:{port}" if subdomains else f"http://localhost:{port}"
url = f"http://admin.archivebox.localhost:{port}" if subdomains else base_url
config = installed_opencode.config
_set_archivebox_config(
config.data_dir,
f"BASE_URL={base_url}",
f"SERVER_SECURITY_MODE={mode}",
)
user = run_archivebox_cmd(
[
"shell",
"-c",
"from django.contrib.auth import get_user_model; User = get_user_model(); User.objects.create_superuser(username='agent-browser-test', password='test-password'); User.objects.create_user(username='agent-regular-test', password='test-password', is_staff=True)",
],
cwd=config.data_dir,
env=config.env,
)
assert user.returncode == 0, user.stderr or user.stdout
process = start_archivebox_server(config.data_dir, port=port, env=config.env, log_name="agent-browser-server.log")
try:
yield url, config.data_dir, process
finally:
if process.poll() is None:
stop_archivebox_process(process)
def _login_cookie(server_url, username="agent-browser-test"):
session = requests.Session()
login_url = server_url + "/admin/login/"
page = session.get(login_url, timeout=10)
assert page.status_code == 200
token = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', page.text)
assert token is not None
response = session.post(
login_url,
data={"username": username, "password": "test-password", "csrfmiddlewaretoken": token[1]},
headers={"Referer": login_url},
allow_redirects=False,
timeout=10,
)
assert response.status_code == 302
return "; ".join(f"{key}={value}" for key, value in session.cookies.items())
def test_agent_websocket_rejects_unauthorized_access(agent_server):
from websockets.asyncio.client import connect
from websockets.exceptions import InvalidStatus
server_url, _, _ = agent_server
admin_cookie = _login_cookie(server_url)
regular_cookie = _login_cookie(server_url, "agent-regular-test")
async def check_denials():
for path, cookie, origin in (
("/admin/agent/opencode/pty/unknown/connect", "", server_url),
("/admin/agent/opencode/pty/unknown/connect", regular_cookie, server_url),
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, "https://untrusted.example"),
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, None),
("/health/", admin_cookie, server_url),
):
with pytest.raises(InvalidStatus) as error:
async with connect(
server_url.replace("http", "ws", 1) + path,
origin=origin,
additional_headers={"Cookie": cookie},
proxy=None,
):
pytest.fail("Unauthorized WebSocket was accepted")
assert error.value.response.status_code == 403
asyncio.run(check_denials())
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
@pytest.mark.parametrize(
"mode,proxy_whitelist,use_cookie,http_status",
[
("unsafe-onedomain-noadmin", "", True, 403),
("safe-onedomain-nojsreplay", "127.0.0.1/32", False, 200),
("safe-onedomain-nojsreplay", "192.0.2.0/24", False, 302),
],
)
def test_agent_websocket_matches_http_security_policy(agent_server, live_opencode, mode, proxy_whitelist, use_cookie, http_status):
from urllib.parse import urlsplit
from websockets.asyncio.client import connect
from websockets.exceptions import InvalidStatus
server_url, data_dir, process = agent_server
cookie = _login_cookie(server_url)
created = requests.post(
server_url + "/admin/agent/opencode/pty",
headers={"Cookie": cookie, "Origin": server_url},
json={"command": "/bin/sh", "args": []},
timeout=10,
)
assert created.status_code == 200
pty_id = created.json()["id"]
stop_archivebox_process(process)
_set_archivebox_config(
data_dir,
f"SERVER_SECURITY_MODE={mode}",
f"REVERSE_PROXY_WHITELIST={proxy_whitelist}",
"REVERSE_PROXY_USER_HEADER=Remote-User",
)
restarted = start_archivebox_server(
data_dir,
port=urlsplit(server_url).port,
env=live_opencode.config.env,
log_name="agent-security-server.log",
)
headers = {"Cookie": cookie} if use_cookie else {"Remote-User": "agent-browser-test"}
try:
response = requests.get(server_url + "/admin/agent/opencode/global/health", headers=headers, allow_redirects=False, timeout=10)
assert response.status_code == http_status
async def check_socket():
connection = connect(
server_url.replace("http", "ws", 1) + f"/admin/agent/opencode/pty/{pty_id}/connect",
origin=server_url,
additional_headers=headers,
proxy=None,
)
if http_status != 200:
with pytest.raises(InvalidStatus) as error:
async with connection:
pytest.fail("Disabled control plane or untrusted proxy accepted a WebSocket")
assert error.value.response.status_code == 403
return
async with connection as socket, asyncio.timeout(10):
await socket.send("printf 'ABX_%s\\n' PROXY_OK\n")
output = ""
while "ABX_PROXY_OK" not in output:
chunk = await socket.recv()
output += chunk.decode() if isinstance(chunk, bytes) else chunk
assert "ABX_PROXY_OK" in output
asyncio.run(check_socket())
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
finally:
deleted = requests.delete(live_opencode.settings["origin"] + f"/pty/{pty_id}", timeout=10)
assert deleted.status_code == 200
stop_archivebox_process(restarted)
def test_agent_preserves_projects_and_survives_storage_failure(agent_server, browser_runtime):
server_url, data_dir, _ = agent_server
script = r"""
const assert = require('node:assert/strict');
const puppeteer = require('puppeteer');
const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
(async () => {
for (const disabled of [false, true]) {
const browser = await puppeteer.launch({
executablePath: config.chrome,
headless: true,
// Headless interaction must not depend on the desktop display clock.
args: ['--no-sandbox', '--disable-frame-rate-limit', ...(disabled ? ['--disable-local-storage'] : [])],
});
try {
const page = await browser.newPage();
await page.goto(new URL('/admin/login/', config.url).href, {waitUntil: 'domcontentloaded'});
await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
await page.locator('#login-form input[name="password"]').fill('test-password');
assert.equal(await page.$eval('#login-form input[name="username"]', element => element.value), 'agent-browser-test');
assert.equal(await page.$eval('#login-form input[name="password"]', element => element.value), 'test-password');
await Promise.all([
page.waitForNavigation({waitUntil: 'domcontentloaded'}),
page.locator('#login-form input[type="submit"]').click(),
]);
assert.ok(!page.url().includes('/admin/login/'), page.url());
assert.equal((await page.goto(config.url, {waitUntil: 'domcontentloaded'})).status(), 200);
const welcome = '#opencode-agent-welcome';
await page.waitForSelector(welcome, {visible: true});
if (disabled) {
assert.equal(await page.evaluate(() => {
try { return localStorage === null; }
catch (error) { if (error.name !== 'SecurityError') throw error; return true; }
}), true);
} else {
await page.click('#opencode-agent-welcome-dismiss');
await page.waitForSelector(welcome, {hidden: true});
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
assert.equal(await page.$eval(welcome, element => element.hidden), true);
const existing = {
list: [{type: 'http', http: {url: 'http://other.example'}}],
projects: {
local: [{worktree: '/other-project', expanded: false}],
'http://other.example': [{worktree: '/remote-project', expanded: true}],
},
lastProject: {'http://other.example': '/remote-project'},
};
await page.evaluate(value => localStorage.setItem('opencode.global.dat:server', JSON.stringify(value)), existing);
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
await page.waitForFunction(workdir => {
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
return state.projects.local?.some(project => project.worktree === workdir);
}, {}, config.workdir);
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('opencode.global.dat:server')));
assert.deepEqual(saved.list, existing.list);
assert.deepEqual(saved.projects['http://other.example'], existing.projects['http://other.example']);
assert.equal(saved.lastProject['http://other.example'], '/remote-project');
const projects = saved.projects.local;
assert.ok(Array.isArray(projects), JSON.stringify({url: page.url(), saved, existing}));
assert.deepEqual(projects.find(project => project.worktree === '/other-project'), {worktree: '/other-project', expanded: false});
assert.equal(projects.filter(project => project.worktree === config.workdir).length, 1);
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
assert.equal(await page.evaluate(workdir => {
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
return state.projects.local.filter(project => project.worktree === workdir).length;
}, config.workdir), 1);
// Fill the actual browser quota; do not replace or intercept Storage methods.
const failure = await page.evaluate(() => {
localStorage.clear();
let low = 0, high = 8 * 1024 * 1024;
while (low < high) {
const size = Math.ceil((low + high) / 2);
try { localStorage.setItem('padding', 'x'.repeat(size)); low = size; }
catch (error) { if (error.name !== 'QuotaExceededError') throw error; high = size - 1; }
}
try { localStorage.setItem('extra', '1'); return null; }
catch (error) { return error.name; }
});
assert.equal(failure, 'QuotaExceededError');
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
await page.waitForSelector(welcome, {visible: true});
}
await page.click('#opencode-agent-welcome-dismiss');
await page.waitForSelector(welcome, {hidden: true});
assert.equal((await page.goto(new URL('/add/', config.url).href, {waitUntil: 'domcontentloaded'})).status(), 200);
} finally {
await browser.close();
}
}
console.log('AGENT_STORAGE_FAILURE_ISOLATED');
})().catch(error => { console.error(error); process.exitCode = 1; });
"""
result = subprocess.run(
[str(browser_runtime["node_binary"]), "-e", script],
input=json.dumps(
{
"chrome": str(browser_runtime["chrome_binary"]),
"url": f"{server_url}/admin/agent",
"workdir": str(data_dir.resolve()),
},
),
env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
text=True,
capture_output=True,
timeout=120,
)
assert result.returncode == 0, result.stderr or result.stdout
assert "AGENT_STORAGE_FAILURE_ISOLATED" in result.stdout
@@ -0,0 +1,90 @@
"""Exercise the real agent wrapper with Chromium's native storage failures."""
import asyncio
import pytest
import requests
from .conftest import start_archivebox_server, stop_archivebox_process
from .test_opencode_agent_1 import _set_archivebox_config
from .test_opencode_agent_1 import installed_opencode as installed_opencode
from .test_opencode_agent_1 import live_opencode as live_opencode
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
from .test_server_security_browser import browser_runtime as browser_runtime
from .test_opencode_browser_1 import (
pytestmark as pytestmark,
agent_server as agent_server,
_login_cookie as _login_cookie,
)
@pytest.mark.parametrize(
"mode,proxy_whitelist,use_cookie,http_status",
[
("unsafe-onedomain-noadmin", "", True, 403),
("safe-onedomain-nojsreplay", "127.0.0.1/32", False, 200),
("safe-onedomain-nojsreplay", "192.0.2.0/24", False, 302),
],
)
def test_agent_websocket_matches_http_security_policy(agent_server, live_opencode, mode, proxy_whitelist, use_cookie, http_status):
from urllib.parse import urlsplit
from websockets.asyncio.client import connect
from websockets.exceptions import InvalidStatus
server_url, data_dir, process = agent_server
cookie = _login_cookie(server_url)
created = requests.post(
server_url + "/admin/agent/opencode/pty",
headers={"Cookie": cookie, "Origin": server_url},
json={"command": "/bin/sh", "args": []},
timeout=10,
)
assert created.status_code == 200
pty_id = created.json()["id"]
stop_archivebox_process(process)
_set_archivebox_config(
data_dir,
f"SERVER_SECURITY_MODE={mode}",
f"REVERSE_PROXY_WHITELIST={proxy_whitelist}",
"REVERSE_PROXY_USER_HEADER=Remote-User",
)
restarted = start_archivebox_server(
data_dir,
port=urlsplit(server_url).port,
env=live_opencode.config.env,
log_name="agent-security-server.log",
)
headers = {"Cookie": cookie} if use_cookie else {"Remote-User": "agent-browser-test"}
try:
response = requests.get(server_url + "/admin/agent/opencode/global/health", headers=headers, allow_redirects=False, timeout=10)
assert response.status_code == http_status
async def check_socket():
connection = connect(
server_url.replace("http", "ws", 1) + f"/admin/agent/opencode/pty/{pty_id}/connect",
origin=server_url,
additional_headers=headers,
proxy=None,
)
if http_status != 200:
with pytest.raises(InvalidStatus) as error:
async with connection:
pytest.fail("Disabled control plane or untrusted proxy accepted a WebSocket")
assert error.value.response.status_code == 403
return
async with connection as socket, asyncio.timeout(10):
await socket.send("printf 'ABX_%s\\n' PROXY_OK\n")
output = ""
while "ABX_PROXY_OK" not in output:
chunk = await socket.recv()
output += chunk.decode() if isinstance(chunk, bytes) else chunk
assert "ABX_PROXY_OK" in output
asyncio.run(check_socket())
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
finally:
deleted = requests.delete(live_opencode.settings["origin"] + f"/pty/{pty_id}", timeout=10)
assert deleted.status_code == 200
stop_archivebox_process(restarted)
+162
View File
@@ -0,0 +1,162 @@
"""Exercise the real agent wrapper with Chromium's native storage failures."""
import asyncio
import json
import os
import subprocess
import pytest
import requests
from .test_opencode_agent_1 import installed_opencode as installed_opencode
from .test_opencode_agent_1 import live_opencode as live_opencode
from .test_opencode_agent_1 import opencode_archive_config as opencode_archive_config
from .test_server_security_browser import browser_runtime as browser_runtime
from .test_opencode_browser_1 import (
pytestmark as pytestmark,
agent_server as agent_server,
_login_cookie as _login_cookie,
)
def test_agent_websocket_rejects_unauthorized_access(agent_server):
from websockets.asyncio.client import connect
from websockets.exceptions import InvalidStatus
server_url, _, _ = agent_server
admin_cookie = _login_cookie(server_url)
regular_cookie = _login_cookie(server_url, "agent-regular-test")
async def check_denials():
for path, cookie, origin in (
("/admin/agent/opencode/pty/unknown/connect", "", server_url),
("/admin/agent/opencode/pty/unknown/connect", regular_cookie, server_url),
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, "https://untrusted.example"),
("/admin/agent/opencode/pty/unknown/connect", admin_cookie, None),
("/health/", admin_cookie, server_url),
):
with pytest.raises(InvalidStatus) as error:
async with connect(
server_url.replace("http", "ws", 1) + path,
origin=origin,
additional_headers={"Cookie": cookie},
proxy=None,
):
pytest.fail("Unauthorized WebSocket was accepted")
assert error.value.response.status_code == 403
asyncio.run(check_denials())
assert requests.get(server_url + "/health/", timeout=10).status_code == 200
def test_agent_preserves_projects_and_survives_storage_failure(agent_server, browser_runtime):
server_url, data_dir, _ = agent_server
script = r"""
const assert = require('node:assert/strict');
const puppeteer = require('puppeteer');
const config = JSON.parse(require('node:fs').readFileSync(0, 'utf8'));
(async () => {
for (const disabled of [false, true]) {
const browser = await puppeteer.launch({
executablePath: config.chrome,
headless: true,
// Headless interaction must not depend on the desktop display clock.
args: ['--no-sandbox', '--disable-frame-rate-limit', ...(disabled ? ['--disable-local-storage'] : [])],
});
try {
const page = await browser.newPage();
await page.goto(new URL('/admin/login/', config.url).href, {waitUntil: 'domcontentloaded'});
await page.locator('#login-form input[name="username"]').fill('agent-browser-test');
await page.locator('#login-form input[name="password"]').fill('test-password');
assert.equal(await page.$eval('#login-form input[name="username"]', element => element.value), 'agent-browser-test');
assert.equal(await page.$eval('#login-form input[name="password"]', element => element.value), 'test-password');
await Promise.all([
page.waitForNavigation({waitUntil: 'domcontentloaded'}),
page.locator('#login-form input[type="submit"]').click(),
]);
assert.ok(!page.url().includes('/admin/login/'), page.url());
assert.equal((await page.goto(config.url, {waitUntil: 'domcontentloaded'})).status(), 200);
const welcome = '#opencode-agent-welcome';
await page.waitForSelector(welcome, {visible: true});
if (disabled) {
assert.equal(await page.evaluate(() => {
try { return localStorage === null; }
catch (error) { if (error.name !== 'SecurityError') throw error; return true; }
}), true);
} else {
await page.click('#opencode-agent-welcome-dismiss');
await page.waitForSelector(welcome, {hidden: true});
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
assert.equal(await page.$eval(welcome, element => element.hidden), true);
const existing = {
list: [{type: 'http', http: {url: 'http://other.example'}}],
projects: {
local: [{worktree: '/other-project', expanded: false}],
'http://other.example': [{worktree: '/remote-project', expanded: true}],
},
lastProject: {'http://other.example': '/remote-project'},
};
await page.evaluate(value => localStorage.setItem('opencode.global.dat:server', JSON.stringify(value)), existing);
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
await page.waitForFunction(workdir => {
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
return state.projects.local?.some(project => project.worktree === workdir);
}, {}, config.workdir);
const saved = await page.evaluate(() => JSON.parse(localStorage.getItem('opencode.global.dat:server')));
assert.deepEqual(saved.list, existing.list);
assert.deepEqual(saved.projects['http://other.example'], existing.projects['http://other.example']);
assert.equal(saved.lastProject['http://other.example'], '/remote-project');
const projects = saved.projects.local;
assert.ok(Array.isArray(projects), JSON.stringify({url: page.url(), saved, existing}));
assert.deepEqual(projects.find(project => project.worktree === '/other-project'), {worktree: '/other-project', expanded: false});
assert.equal(projects.filter(project => project.worktree === config.workdir).length, 1);
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
assert.equal(await page.evaluate(workdir => {
const state = JSON.parse(localStorage.getItem('opencode.global.dat:server'));
return state.projects.local.filter(project => project.worktree === workdir).length;
}, config.workdir), 1);
// Fill the actual browser quota; do not replace or intercept Storage methods.
const failure = await page.evaluate(() => {
localStorage.clear();
let low = 0, high = 8 * 1024 * 1024;
while (low < high) {
const size = Math.ceil((low + high) / 2);
try { localStorage.setItem('padding', 'x'.repeat(size)); low = size; }
catch (error) { if (error.name !== 'QuotaExceededError') throw error; high = size - 1; }
}
try { localStorage.setItem('extra', '1'); return null; }
catch (error) { return error.name; }
});
assert.equal(failure, 'QuotaExceededError');
await page.goto(config.url, {waitUntil: 'domcontentloaded'});
await page.waitForSelector(welcome, {visible: true});
}
await page.click('#opencode-agent-welcome-dismiss');
await page.waitForSelector(welcome, {hidden: true});
assert.equal((await page.goto(new URL('/add/', config.url).href, {waitUntil: 'domcontentloaded'})).status(), 200);
} finally {
await browser.close();
}
}
console.log('AGENT_STORAGE_FAILURE_ISOLATED');
})().catch(error => { console.error(error); process.exitCode = 1; });
"""
result = subprocess.run(
[str(browser_runtime["node_binary"]), "-e", script],
input=json.dumps(
{
"chrome": str(browser_runtime["chrome_binary"]),
"url": f"{server_url}/admin/agent",
"workdir": str(data_dir.resolve()),
},
),
env={**os.environ, "NODE_PATH": browser_runtime["node_path"]},
text=True,
capture_output=True,
timeout=120,
)
assert result.returncode == 0, result.stderr or result.stdout
assert "AGENT_STORAGE_FAILURE_ISOLATED" in result.stdout
+6 -1
View File
@@ -91,7 +91,12 @@ def test_tlsnotary_preview_verifies_saved_evidence_without_remote_requests(snaps
assert "Signed response byte range" in topology.inner_text()
assert topology.locator('a[href*="response.http"]').count() > 0
assert topology.locator('a[href*="receipt.json"]').count() > 0
assert page.locator("a", has_text="Verify independently").get_attribute("href") == "https://tlsnotary.zervice.io/"
# The installed plugin owns its verifier domain. Compare its template
# with its declared default so a plugin release does not require a
# simultaneous core test edit, while stale template links still fail.
schema = json.loads((Path(get_plugins_dir()) / "tlsnotary" / "config.json").read_text())
verifier_url = schema["properties"]["TLSNOTARY_VERIFIER_URL"]["default"]
assert page.locator("a", has_text="Verify independently").get_attribute("href") == f"{verifier_url.rstrip('/')}/"
assert all(urlsplit(request).hostname == hostname for request in requests)
page.screenshot(path=str(tmp_path / "tlsnotary-desktop.png"), full_page=True)
page.set_viewport_size({"width": 390, "height": 844})
+266
View File
@@ -0,0 +1,266 @@
#!/usr/bin/env python3
"""Integration tests for recursive crawling functionality."""
import json
import os
from pathlib import Path
import pytest
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.conftest import run_archivebox_cmd
from archivebox.tests.test_orm_helpers import use_archivebox_db
pytestmark = pytest.mark.django_db(transaction=True)
def run_add_until(args, env, condition, timeout=120):
assert args[0] == "archivebox"
result = run_archivebox_cmd(
args[1:],
cwd=Path.cwd(),
env=env,
timeout=timeout,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db("."):
assert condition(), f"Condition was false after command completed: {' '.join(args)}"
return result.stdout, result.stderr
@pytest.mark.timeout(1200)
def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
initialized_archive,
free_tcp_port_factory,
recursive_test_site,
):
"""Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
from archivebox.plugins.discovery import get_plugin_catalog
root_url = recursive_test_site["root_url"]
plugin_selection = ",".join(sorted(plugin for plugin in get_plugin_catalog() if not plugin.startswith("claude")))
env = os.environ.copy()
for preinstalled_path_key in (
"CHROME_BINARY",
"LIB_DIR",
"DATA_DIR",
"NODE_MODULES_DIR",
"NODE_PATH",
"PNPM_BIN_DIR",
"NPM_BIN_DIR",
"CHROMEWEBSTORE_EXTENSIONS_DIR",
):
env.pop(preinstalled_path_key, None)
env.update(
{
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
"CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
"TIMEOUT": "90",
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
"SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
"SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
"CHROME_HEADLESS": "true",
"CHROME_SANDBOX": "false",
"CHROME_ISOLATION": "snapshot",
"LITEPARSE_OCR_ENABLED": "false",
"LITEPARSE_MAX_SOURCES": "4",
},
)
result = run_archivebox_cmd(
[
"add",
"--depth=2",
"--max-urls=8",
"--crawl-max-size=100mb",
"--tag=recursive-all-plugins",
"--parser=url_list",
f"--plugins={plugin_selection}",
root_url,
],
cwd=initialized_archive,
env=env,
timeout=1200,
)
stdout, stderr = result.stdout, result.stderr
if stderr:
print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n")
if stdout:
print(f"\n=== STDOUT (last 4000 chars) ===\n{stdout[-4000:]}\n=== END STDOUT ===\n")
assert result.returncode == 0, stderr or stdout
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get(tags_str="recursive-all-plugins")
snapshots = list(
Snapshot.objects.filter(crawl=crawl)
.order_by("depth", "url")
.values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"),
)
archive_results = list(
ArchiveResult.objects.filter(snapshot__crawl=crawl)
.select_related("snapshot")
.order_by("snapshot__depth", "snapshot__url", "plugin", "hook_name")
.values_list(
"snapshot_id",
"snapshot__url",
"snapshot__depth",
"plugin",
"hook_name",
"status",
"output_files",
"output_size",
"output_str",
),
)
process_snapshot_ids = {
process_id: str(snapshot_id)
for snapshot_id, process_id in ArchiveResult.objects.filter(
snapshot__crawl=crawl,
process_id__isnull=False,
).values_list("snapshot_id", "process_id")
}
processes = list(
Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids)
.order_by("started_at")
.values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
)
assert crawl.max_depth == 2
assert crawl.config["CRAWL_MAX_URLS"] == 8
assert crawl.config["CRAWL_MAX_SIZE"] == 100 * 1024 * 1024
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
expected_urls = {
root_url,
*recursive_test_site["child_urls"],
*recursive_test_site["deep_urls"],
}
snapshots_by_url = {url: (snapshot_id, depth, parent_id) for snapshot_id, url, depth, _status, parent_id, _downloaded_at in snapshots}
assert set(snapshots_by_url) == expected_urls
root_id, root_depth, root_parent_id = snapshots_by_url[root_url]
assert root_depth == 0
assert root_parent_id is None
child_ids_by_url = {}
for child_url in recursive_test_site["child_urls"]:
child_id, child_depth, child_parent_id = snapshots_by_url[child_url]
assert child_depth == 1
assert child_parent_id == root_id
child_ids_by_url[child_url] = child_id
for child_url, deep_url in zip(recursive_test_site["child_urls"], recursive_test_site["deep_urls"], strict=True):
_deep_id, deep_depth, deep_parent_id = snapshots_by_url[deep_url]
assert deep_depth == 2
assert deep_parent_id == child_ids_by_url[child_url]
assert all(status == Snapshot.StatusChoices.SEALED for _id, _url, _depth, status, _parent, _downloaded_at in snapshots)
assert all(downloaded_at is not None for _id, _url, _depth, _status, _parent, downloaded_at in snapshots)
assert archive_results
allowed_statuses = {
ArchiveResult.StatusChoices.SUCCEEDED,
ArchiveResult.StatusChoices.NORESULTS,
ArchiveResult.StatusChoices.SKIPPED,
}
unexpected_results = [
{
"url": url,
"depth": depth,
"plugin": plugin,
"hook_name": hook_name,
"status": status,
"output_str": output_str,
}
for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
]
# This fixture serves HTTP. TLSNotary must reject every plaintext document;
# a setup, timeout, or verifier error must not count as that expected refusal.
tlsnotary_results = [row for row in archive_results if row[3] == "tlsnotary"]
assert {row[1] for row in tlsnotary_results} == expected_urls
assert len(tlsnotary_results) == len(expected_urls)
with use_archivebox_db(initialized_archive):
for snapshot_id, _url, _depth, _plugin, hook_name, status, files, size, _output in tlsnotary_results:
assert status == ArchiveResult.StatusChoices.FAILED
assert not files
assert size == 0
process = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="tlsnotary", hook_name=hook_name).process
assert process is not None
assert process.status == Process.StatusChoices.EXITED
assert process.exit_code == 1
assert process.stderr.strip() == "[tlsnotary] TLSNotary requires an HTTPS document"
assert not [result for result in unexpected_results if result["plugin"] != "tlsnotary"]
ytdlp_results = [
(url, status, output_str)
for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
if plugin == "ytdlp"
]
assert ytdlp_results
assert all(status != ArchiveResult.StatusChoices.FAILED for _url, status, _output_str in ytdlp_results), ytdlp_results
plugins_seen = {plugin for _snapshot_id, _url, _depth, plugin, _hook_name, _status, _files, _size, _output in archive_results}
assert {
"wget",
"headers",
"title",
"pdf",
"screenshot",
"dom",
"singlefile",
"readability",
"mercury",
"htmltotext",
"favicon",
"parse_html_urls",
"archivedotorg",
}.issubset(plugins_seen)
snapshot_root = initialized_archive / "archive/users/system/snapshots"
assert list(snapshot_root.rglob("wget/**/*.html"))
assert list(snapshot_root.rglob("headers/**/headers.json"))
assert list(snapshot_root.rglob("title/title.txt"))
assert list(snapshot_root.rglob("pdf/**/*.pdf"))
assert list(snapshot_root.rglob("screenshot/**/*.png"))
assert list(snapshot_root.rglob("dom/**/*.html"))
assert list(snapshot_root.rglob("singlefile/**/*.html"))
assert list(snapshot_root.rglob("readability/**/*.html"))
assert list(snapshot_root.rglob("mercury/**/*.html"))
assert list(snapshot_root.rglob("htmltotext/**/*.txt"))
assert list(snapshot_root.rglob("favicon/**/*"))
urls_jsonl_files = list(snapshot_root.rglob("parse_html_urls/urls.jsonl"))
assert urls_jsonl_files
parsed_urls = {
json.loads(line)["url"] for path in urls_jsonl_files for line in path.read_text(errors="ignore").splitlines() if line.strip()
}
assert set(recursive_test_site["child_urls"]).issubset(parsed_urls)
assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
assert processes
assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
intervals = []
for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
if not started_at or not ended_at:
continue
process_snapshot_id = process_snapshot_ids.get(process_id)
if process_snapshot_id is None:
continue
intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))
overlapping = [
(left, right)
for index, left in enumerate(intervals)
for right in intervals[index + 1 :]
if left[0] != right[0] and left[1] < right[2] and right[1] < left[2]
]
assert overlapping, f"Expected hook processes from different snapshots to overlap, got intervals: {intervals}"
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""Integration tests for recursive crawling functionality."""
import os
import pytest
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.conftest import run_archivebox_cmd
from archivebox.tests.test_orm_helpers import use_archivebox_db
from .test_recursive_crawl_1 import (
pytestmark as pytestmark,
run_add_until as run_add_until,
)
@pytest.mark.timeout(600)
def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(initialized_archive, recursive_test_site):
"""A real three-URL capture must not launch all Chrome hook trees at once."""
urls = [recursive_test_site["root_url"], *recursive_test_site["child_urls"][:2]]
env = os.environ.copy()
env.update(
{
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
"CHROME_HEADLESS": "true",
"CHROME_SANDBOX": "false",
"CHROME_ISOLATION": "crawl",
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
"TIMEOUT": "90",
},
)
result = run_archivebox_cmd(
["add", "--parser=url_list", "--plugins=chrome,wget,screenshot", "--tag=measured-admission", *urls],
cwd=initialized_archive,
env=env,
timeout=600,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get(tags_str="measured-admission")
snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("created_at"))
hook_intervals = list(
Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
archiveresult__snapshot__crawl=crawl,
)
.values_list("archiveresult__snapshot_id", "started_at", "ended_at")
.distinct(),
)
assert len(snapshots) == 3
assert all(snapshot.status == Snapshot.StatusChoices.SEALED and snapshot.downloaded_at for snapshot in snapshots)
first_id = snapshots[0].id
first_end = snapshots[0].downloaded_at
later_starts = [started for snapshot_id, started, _ended in hook_intervals if snapshot_id != first_id and started]
assert later_starts and first_end <= min(later_starts), hook_intervals
later_intervals = [
(snapshot_id, started, ended) for snapshot_id, started, ended in hook_intervals if snapshot_id != first_id and started and ended
]
assert any(
left_id != right_id and left_start < right_end and right_start < left_end
for index, (left_id, left_start, left_end) in enumerate(later_intervals)
for right_id, right_start, right_end in later_intervals[index + 1 :]
), later_intervals
def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, initialized_archive):
"""Test that Snapshot model has parent_snapshot field."""
column_names = {field.column for field in Snapshot._meta.local_fields}
assert "parent_snapshot_id" in column_names, f"Snapshot table should have parent_snapshot_id column. Columns: {column_names}"
@@ -13,21 +13,10 @@ from archivebox.machine.models import Binary, Process
from archivebox.tests.conftest import run_archivebox_cmd, cli_env
from archivebox.tests.test_orm_helpers import use_archivebox_db
pytestmark = pytest.mark.django_db(transaction=True)
def run_add_until(args, env, condition, timeout=120):
assert args[0] == "archivebox"
result = run_archivebox_cmd(
args[1:],
cwd=Path.cwd(),
env=env,
timeout=timeout,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db("."):
assert condition(), f"Condition was false after command completed: {' '.join(args)}"
return result.stdout, result.stderr
from .test_recursive_crawl_1 import (
pytestmark as pytestmark,
run_add_until as run_add_until,
)
def test_background_hooks_dont_block_parser_extractors(tmp_path, initialized_archive, recursive_test_site):
@@ -492,300 +481,6 @@ def test_add_archivewebpage_installs_required_chrome_dependency(initialized_arch
)
@pytest.mark.timeout(1200)
def test_recursive_crawl_depth_two_all_plugins_runs_snapshots_in_parallel(
initialized_archive,
free_tcp_port_factory,
recursive_test_site,
):
"""Run a bounded real depth=2 crawl with all plugins enabled and verify parallel snapshot execution."""
from archivebox.plugins.discovery import get_plugin_catalog
root_url = recursive_test_site["root_url"]
plugin_selection = ",".join(sorted(plugin for plugin in get_plugin_catalog() if not plugin.startswith("claude")))
env = os.environ.copy()
for preinstalled_path_key in (
"CHROME_BINARY",
"LIB_DIR",
"DATA_DIR",
"NODE_MODULES_DIR",
"NODE_PATH",
"PNPM_BIN_DIR",
"NPM_BIN_DIR",
"CHROMEWEBSTORE_EXTENSIONS_DIR",
):
env.pop(preinstalled_path_key, None)
env.update(
{
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
"CHROMEWEBSTORE_EXTENSIONS_DIR": str(initialized_archive / "lib/chromewebstore/extensions"),
"TIMEOUT": "90",
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
"SEARCH_BACKEND_SONIC_HOST_NAME": "127.0.0.1",
"SEARCH_BACKEND_SONIC_PORT": str(free_tcp_port_factory()),
"CHROME_HEADLESS": "true",
"CHROME_SANDBOX": "false",
"CHROME_ISOLATION": "snapshot",
"LITEPARSE_OCR_ENABLED": "false",
"LITEPARSE_MAX_SOURCES": "4",
},
)
result = run_archivebox_cmd(
[
"add",
"--depth=2",
"--max-urls=8",
"--crawl-max-size=100mb",
"--tag=recursive-all-plugins",
"--parser=url_list",
f"--plugins={plugin_selection}",
root_url,
],
cwd=initialized_archive,
env=env,
timeout=1200,
)
stdout, stderr = result.stdout, result.stderr
if stderr:
print(f"\n=== STDERR ===\n{stderr}\n=== END STDERR ===\n")
if stdout:
print(f"\n=== STDOUT (last 4000 chars) ===\n{stdout[-4000:]}\n=== END STDOUT ===\n")
assert result.returncode == 0, stderr or stdout
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get(tags_str="recursive-all-plugins")
snapshots = list(
Snapshot.objects.filter(crawl=crawl)
.order_by("depth", "url")
.values_list("id", "url", "depth", "status", "parent_snapshot_id", "downloaded_at"),
)
archive_results = list(
ArchiveResult.objects.filter(snapshot__crawl=crawl)
.select_related("snapshot")
.order_by("snapshot__depth", "snapshot__url", "plugin", "hook_name")
.values_list(
"snapshot_id",
"snapshot__url",
"snapshot__depth",
"plugin",
"hook_name",
"status",
"output_files",
"output_size",
"output_str",
),
)
process_snapshot_ids = {
process_id: str(snapshot_id)
for snapshot_id, process_id in ArchiveResult.objects.filter(
snapshot__crawl=crawl,
process_id__isnull=False,
).values_list("snapshot_id", "process_id")
}
processes = list(
Process.objects.filter(process_type=Process.TypeChoices.HOOK, id__in=process_snapshot_ids)
.order_by("started_at")
.values_list("id", "pwd", "cmd", "status", "exit_code", "started_at", "ended_at"),
)
assert crawl.max_depth == 2
assert crawl.config["CRAWL_MAX_URLS"] == 8
assert crawl.config["CRAWL_MAX_SIZE"] == 100 * 1024 * 1024
assert crawl.config["CRAWL_MAX_CONCURRENT_SNAPSHOTS"] == 3
assert crawl.status == Crawl.StatusChoices.SEALED
assert crawl.retry_at is None
expected_urls = {
root_url,
*recursive_test_site["child_urls"],
*recursive_test_site["deep_urls"],
}
snapshots_by_url = {url: (snapshot_id, depth, parent_id) for snapshot_id, url, depth, _status, parent_id, _downloaded_at in snapshots}
assert set(snapshots_by_url) == expected_urls
root_id, root_depth, root_parent_id = snapshots_by_url[root_url]
assert root_depth == 0
assert root_parent_id is None
child_ids_by_url = {}
for child_url in recursive_test_site["child_urls"]:
child_id, child_depth, child_parent_id = snapshots_by_url[child_url]
assert child_depth == 1
assert child_parent_id == root_id
child_ids_by_url[child_url] = child_id
for child_url, deep_url in zip(recursive_test_site["child_urls"], recursive_test_site["deep_urls"], strict=True):
_deep_id, deep_depth, deep_parent_id = snapshots_by_url[deep_url]
assert deep_depth == 2
assert deep_parent_id == child_ids_by_url[child_url]
assert all(status == Snapshot.StatusChoices.SEALED for _id, _url, _depth, status, _parent, _downloaded_at in snapshots)
assert all(downloaded_at is not None for _id, _url, _depth, _status, _parent, downloaded_at in snapshots)
assert archive_results
allowed_statuses = {
ArchiveResult.StatusChoices.SUCCEEDED,
ArchiveResult.StatusChoices.NORESULTS,
ArchiveResult.StatusChoices.SKIPPED,
}
unexpected_results = [
{
"url": url,
"depth": depth,
"plugin": plugin,
"hook_name": hook_name,
"status": status,
"output_str": output_str,
}
for _snapshot_id, url, depth, plugin, hook_name, status, _files, _size, output_str in archive_results
if not (status in allowed_statuses or (plugin == "archivedotorg" and status == ArchiveResult.StatusChoices.FAILED))
]
# This fixture serves HTTP. TLSNotary must reject every plaintext document;
# a setup, timeout, or verifier error must not count as that expected refusal.
tlsnotary_results = [row for row in archive_results if row[3] == "tlsnotary"]
assert {row[1] for row in tlsnotary_results} == expected_urls
assert len(tlsnotary_results) == len(expected_urls)
with use_archivebox_db(initialized_archive):
for snapshot_id, _url, _depth, _plugin, hook_name, status, files, size, _output in tlsnotary_results:
assert status == ArchiveResult.StatusChoices.FAILED
assert not files
assert size == 0
process = ArchiveResult.objects.get(snapshot_id=snapshot_id, plugin="tlsnotary", hook_name=hook_name).process
assert process is not None
assert process.status == Process.StatusChoices.EXITED
assert process.exit_code == 1
assert process.stderr.strip() == "[tlsnotary] TLSNotary requires an HTTPS document"
assert not [result for result in unexpected_results if result["plugin"] != "tlsnotary"]
ytdlp_results = [
(url, status, output_str)
for _snapshot_id, url, _depth, plugin, _hook_name, status, _files, _size, output_str in archive_results
if plugin == "ytdlp"
]
assert ytdlp_results
assert all(status != ArchiveResult.StatusChoices.FAILED for _url, status, _output_str in ytdlp_results), ytdlp_results
plugins_seen = {plugin for _snapshot_id, _url, _depth, plugin, _hook_name, _status, _files, _size, _output in archive_results}
assert {
"wget",
"headers",
"title",
"pdf",
"screenshot",
"dom",
"singlefile",
"readability",
"mercury",
"htmltotext",
"favicon",
"parse_html_urls",
"archivedotorg",
}.issubset(plugins_seen)
snapshot_root = initialized_archive / "archive/users/system/snapshots"
assert list(snapshot_root.rglob("wget/**/*.html"))
assert list(snapshot_root.rglob("headers/**/headers.json"))
assert list(snapshot_root.rglob("title/title.txt"))
assert list(snapshot_root.rglob("pdf/**/*.pdf"))
assert list(snapshot_root.rglob("screenshot/**/*.png"))
assert list(snapshot_root.rglob("dom/**/*.html"))
assert list(snapshot_root.rglob("singlefile/**/*.html"))
assert list(snapshot_root.rglob("readability/**/*.html"))
assert list(snapshot_root.rglob("mercury/**/*.html"))
assert list(snapshot_root.rglob("htmltotext/**/*.txt"))
assert list(snapshot_root.rglob("favicon/**/*"))
urls_jsonl_files = list(snapshot_root.rglob("parse_html_urls/urls.jsonl"))
assert urls_jsonl_files
parsed_urls = {
json.loads(line)["url"] for path in urls_jsonl_files for line in path.read_text(errors="ignore").splitlines() if line.strip()
}
assert set(recursive_test_site["child_urls"]).issubset(parsed_urls)
assert set(recursive_test_site["deep_urls"]).issubset(parsed_urls)
assert processes
assert all(status == Process.StatusChoices.EXITED for _id, _pwd, _cmd, status, _exit_code, _started_at, _ended_at in processes)
intervals = []
for process_id, pwd, cmd, _status, _exit_code, started_at, ended_at in processes:
if not started_at or not ended_at:
continue
process_snapshot_id = process_snapshot_ids.get(process_id)
if process_snapshot_id is None:
continue
intervals.append((process_snapshot_id, started_at, ended_at, pwd, cmd))
overlapping = [
(left, right)
for index, left in enumerate(intervals)
for right in intervals[index + 1 :]
if left[0] != right[0] and left[1] < right[2] and right[1] < left[2]
]
assert overlapping, f"Expected hook processes from different snapshots to overlap, got intervals: {intervals}"
@pytest.mark.timeout(600)
def test_direct_url_crawl_measures_first_snapshot_before_parallel_admission(initialized_archive, recursive_test_site):
"""A real three-URL capture must not launch all Chrome hook trees at once."""
urls = [recursive_test_site["root_url"], *recursive_test_site["child_urls"][:2]]
env = os.environ.copy()
env.update(
{
"USE_COLOR": "false",
"SHOW_PROGRESS": "false",
"URL_ALLOWLIST": r"127\.0\.0\.1[:/].*",
"ABXPKG_LIB_DIR": str(initialized_archive / "lib"),
"CHROME_HEADLESS": "true",
"CHROME_SANDBOX": "false",
"CHROME_ISOLATION": "crawl",
"CRAWL_MAX_CONCURRENT_SNAPSHOTS": "3",
"TIMEOUT": "90",
},
)
result = run_archivebox_cmd(
["add", "--parser=url_list", "--plugins=chrome,wget,screenshot", "--tag=measured-admission", *urls],
cwd=initialized_archive,
env=env,
timeout=600,
)
assert result.returncode == 0, result.stderr or result.stdout
with use_archivebox_db(initialized_archive):
crawl = Crawl.objects.get(tags_str="measured-admission")
snapshots = list(Snapshot.objects.filter(crawl=crawl).order_by("created_at"))
hook_intervals = list(
Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
archiveresult__snapshot__crawl=crawl,
)
.values_list("archiveresult__snapshot_id", "started_at", "ended_at")
.distinct(),
)
assert len(snapshots) == 3
assert all(snapshot.status == Snapshot.StatusChoices.SEALED and snapshot.downloaded_at for snapshot in snapshots)
first_id = snapshots[0].id
first_end = snapshots[0].downloaded_at
later_starts = [started for snapshot_id, started, _ended in hook_intervals if snapshot_id != first_id and started]
assert later_starts and first_end <= min(later_starts), hook_intervals
later_intervals = [
(snapshot_id, started, ended) for snapshot_id, started, ended in hook_intervals if snapshot_id != first_id and started and ended
]
assert any(
left_id != right_id and left_start < right_end and right_start < left_end
for index, (left_id, left_start, left_end) in enumerate(later_intervals)
for right_id, right_start, right_end in later_intervals[index + 1 :]
), later_intervals
def test_crawl_snapshot_has_parent_snapshot_field(tmp_path, initialized_archive):
"""Test that Snapshot model has parent_snapshot field."""
column_names = {field.column for field in Snapshot._meta.local_fields}
assert "parent_snapshot_id" in column_names, f"Snapshot table should have parent_snapshot_id column. Columns: {column_names}"
def test_snapshot_depth_field_exists(tmp_path, initialized_archive):
"""Test that Snapshot model has depth field."""
@@ -857,7 +552,3 @@ def test_archiveresult_worker_queue_filters_by_foreground_extractors(tmp_path, i
assert len(non_queued) > 0 or len(parser_status) == 0, (
f"With {len(bg_results)} background hooks started, parser extractors should still run. Got statuses: {parser_statuses}"
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+99 -7
View File
@@ -14,7 +14,7 @@ import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
RELEASE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release.yml"
RELEASE_CANDIDATE_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "release-candidate.yml"
CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml"
PIP_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pip.yml"
DOCKER_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "docker.yml"
VERIFY_STAGING_SCRIPT = REPO_ROOT / "bin" / "verify-staging-release.py"
@@ -115,9 +115,9 @@ def test_stable_release_reconciles_concurrent_version_only_dev_bump(tmp_path, de
assert result.returncode == 0, result.stderr
merged = _git(work, "ls-remote", "origin", "refs/heads/dev").split()[0]
channel_step = next(
step for step in yaml.safe_load(RELEASE_CANDIDATE_WORKFLOW.read_text())["jobs"]["prepare"]["steps"] if step.get("id") == "channel"
)
prepare_path = yaml.safe_load(CI_WORKFLOW.read_text())["jobs"]["prepare"]["uses"]
prepare = yaml.safe_load((REPO_ROOT / prepare_path).read_text())
channel_step = next(step for step in prepare["jobs"]["prepare"]["steps"] if step.get("id") == "channel")
def stable_channel(sha, branch="dev"):
output = tmp_path / "channel-output"
@@ -140,7 +140,7 @@ def test_stable_release_reconciles_concurrent_version_only_dev_bump(tmp_path, de
assert _git(work, "rev-parse", f"{merged}^{{tree}}") == _git(work, "rev-parse", f"{stable}^{{tree}}")
assert _git(work, "merge-base", stable, candidate) == base
# Reconciliation creates a new merge commit with the tested main tree. Its
# candidate workflow must not immediately bump dev and undo synchronization.
# version preparation must not immediately bump dev and undo synchronization.
assert stable_channel(merged) == "stable=true"
assert stable_channel(merged, branch="main") == "stable=false"
@@ -184,8 +184,28 @@ def test_release_uses_registered_publisher_and_authorized_tag_credentials():
assert jobs["cascade"]["if"] == "needs.python-release.outputs.release_ready == 'true'"
assert all(step.get("name") != "Verify published PyPI package installs and runs" for step in python_release["steps"])
candidate = yaml.safe_load(RELEASE_CANDIDATE_WORKFLOW.read_text())
assert candidate["jobs"]["python-artifacts"]["with"]["full_tests"] is False
ci = yaml.safe_load(CI_WORKFLOW.read_text())
screenshots = yaml.safe_load((REPO_ROOT / ".github" / "workflows" / "screenshots.yml").read_text())
prepare_path = ci["jobs"]["prepare"]["uses"]
assert screenshots["jobs"]["prepare"]["uses"] == prepare_path
prepare = yaml.safe_load((REPO_ROOT / prepare_path).read_text())
assert prepare["jobs"]["prepare"]["steps"][-1]["uses"] == "ArchiveBox/monorepo/.github/actions/prepare-release-version@main"
assert screenshots["jobs"]["deploy"]["needs"] == "prepare"
assert screenshots["jobs"]["deploy"]["if"] == "needs.prepare.outputs.run_jobs == 'true'"
for name, job in ci["jobs"].items():
if name not in {"prepare", "required"}:
assert job["needs"] == "prepare"
assert job["if"] == "needs.prepare.outputs.run_jobs == 'true'"
gate = next(step["run"] for step in jobs["candidate"]["steps"] if step.get("id") == "verified")
assert 'ARTIFACT_RUN_ID="$CI_RUN_ID"' in gate
assert 'DIGEST_RUN_ID="$CI_RUN_ID"' in gate
assert 'ARTIFACT_RUN_ID="$CANDIDATE_RUN_ID"' not in gate
assert 'DIGEST_RUN_ID="$CANDIDATE_RUN_ID"' not in gate
docker_workflow = yaml.safe_load(DOCKER_WORKFLOW.read_text())
version_check = next(
step for step in docker_workflow["jobs"]["build"]["steps"] if step.get("name") == "Validate exact built image version and commit"
)
assert version_check["if"] == "inputs.full_tests"
pip_workflow = yaml.safe_load(PIP_WORKFLOW.read_text())
install_script = next(
step["run"] for step in pip_workflow["jobs"]["build"]["steps"] if step.get("name") == "Release wheel import and CLI smoke"
@@ -415,3 +435,75 @@ def test_staging_acceptance_limits_only_exact_known_auth_and_tls_failures():
assert "missing or truncated" in embedded.group(1)
assert "output.stat().st_size != metadata.get('size')" in embedded.group(1)
assert "tlsnotary reported unexpected status" in embedded.group(1)
@pytest.mark.parametrize("requested", ["", "archivebox/tests/test_urls.py", "archivebox/tests/test_release_workflow.py"])
def test_ci_batches_preserve_inventory_runner_ownership_and_single_file_runs(tmp_path, requested):
hosted_paths = {
path.relative_to(REPO_ROOT).as_posix()
for path in (REPO_ROOT / "archivebox/tests").glob("test_*.py")
if "# ci-runner: hosted" in path.read_text().splitlines()[:5]
}
result = subprocess.run(
["uv", "run", "--no-cache", "--no-project", "python", str(REPO_ROOT / ".github/scripts/discover_test_matrix.py")],
cwd=tmp_path,
env={**os.environ, "TEST_FILE": requested, "UGNAS_CI_MAX_JOBS": "3"},
capture_output=True,
text=True,
check=True,
)
matrix = json.loads(result.stdout.splitlines()[1])
paths = [path for item in matrix for path in item["paths"]]
expected = sorted(path.relative_to(REPO_ROOT).as_posix() for path in (REPO_ROOT / "archivebox/tests").glob("test_*.py"))
assert sorted(paths) == expected
assert len(paths) == len(set(paths))
assert len(matrix) < len(paths)
assert all(1 <= len(item["paths"]) <= 8 for item in matrix)
assert sum(item["ugnas"] for item in matrix) == 3
assert not any(path in hosted_paths for item in matrix if item["ugnas"] for path in item["paths"])
ldap = next(item for item in matrix if item["extra"] == "ldap")
assert ldap["paths"] == ["archivebox/tests/test_auth_ldap.py"]
if requested:
assert len([item for item in matrix if item["paths"] == [requested]]) == 1
def test_ci_discovery_handles_added_renamed_and_deleted_files_without_timing_edits(tmp_path):
script = tmp_path / ".github/scripts/discover_test_matrix.py"
script.parent.mkdir(parents=True)
shutil.copy2(REPO_ROOT / ".github/scripts/discover_test_matrix.py", script)
shutil.copy2(REPO_ROOT / ".github/test-durations.json", tmp_path / ".github/test-durations.json")
tests = tmp_path / "archivebox/tests"
tests.mkdir(parents=True)
for name in ("test_core_config.py", "test_util.py"):
shutil.copy2(REPO_ROOT / "archivebox/tests" / name, tests / name)
def discover():
result = subprocess.run(
["uv", "run", "--no-cache", "--no-project", "python", str(script)],
cwd=tmp_path,
env={**os.environ, "TEST_FILE": "", "UGNAS_CI_MAX_JOBS": "3"},
capture_output=True,
text=True,
check=True,
)
matrix = json.loads(result.stdout.splitlines()[1])
paths = [path for item in matrix for path in item["paths"]]
assert sorted(paths) == sorted(path.relative_to(tmp_path).as_posix() for path in tests.glob("test_*.py"))
assert len(paths) == len(set(paths))
return matrix
assert len(discover()) == 1
(tests / "test_util.py").rename(tests / "test_renamed.py")
assert len(discover()) == 2
shutil.copy2(REPO_ROOT / "archivebox/tests/test_util.py", tests / "test_added.py")
assert len(discover()) == 3
(tests / "test_core_config.py").unlink()
assert len(discover()) == 2
source = REPO_ROOT / "archivebox/tests/test_util.py"
(tests / "test_runner.py").write_text("# ci-runner: hosted\n" + source.read_text())
assert next(item for item in discover() if "archivebox/tests/test_runner.py" in item["paths"])["ugnas"] is False
(tests / "test_runner.py").rename(tests / "test_runner_renamed.py")
assert next(item for item in discover() if "archivebox/tests/test_runner_renamed.py" in item["paths"])["ugnas"] is False
(tests / "test_runner_renamed.py").unlink()
assert all(item["ugnas"] for item in discover())
+18 -4
View File
@@ -594,7 +594,11 @@ class TestPublicIndexSearch:
assert b"Public Example Website" in response.content
assert b"No snapshots found." not in response.content
def test_public_index_shows_exact_total_count_and_page_count_for_100_plus_snapshots(self, client, crawl, public_snapshot):
@pytest.mark.django_db(transaction=True)
def test_public_index_shows_cached_total_count_and_page_count_for_100_plus_snapshots(self, client, crawl, public_snapshot):
import time
from django.core.cache import cache
from archivebox.core.models import Snapshot
base_time = public_snapshot.bookmarked_at + timedelta(seconds=1)
@@ -628,11 +632,21 @@ class TestPublicIndexSearch:
],
)
key = "progress-collection:public"
cache.delete_many([key, f"{key}:refresh"])
initial_response = client.get("/public/", HTTP_HOST=WEB_HOST)
assert initial_response.status_code == 200
assert initial_response.context["public_snapshot_count"] is None
assert b"last &raquo;" not in initial_response.content
deadline = time.monotonic() + 5
while cache.get(key) is None and time.monotonic() < deadline:
time.sleep(0.01)
assert cache.get(key)["snapshots"] == 125
response = client.get("/public/", HTTP_HOST=WEB_HOST)
assert response.status_code == 200
assert response.context["paginator"].count == 125
assert response.context["paginator"].num_pages == 3
assert response.context["public_snapshot_count"] == 125
assert response.context["public_page_count"] == 3
assert response.context["page_obj"].has_next() is True
assert len(response.context["object_list"]) == 50
content = response.content.decode()
@@ -1329,7 +1343,7 @@ class TestSearchBackendsE2E:
surface_name,
search_mode,
first_positive_elapsed,
counts[:10],
count_events[:10],
)
assert total_elapsed < 2.0, (surface_name, search_mode, total_elapsed, counts[-10:])
@@ -954,6 +954,17 @@ def test_server_security_modes_in_chrome(
def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archive: Path, browser_runtime, tmp_path: Path) -> None:
from archivebox.core.routes_util import get_snapshot_subdomain
node_binary = Path(browser_runtime["node_binary"])
node_target = node_binary.resolve(strict=True)
node_version = subprocess.check_output([str(node_binary), "--version"], text=True).strip()
def assert_browser_runtime_unchanged(stage):
actual_target = node_binary.resolve(strict=True)
actual_version = subprocess.check_output([str(node_binary), "--version"], text=True).strip()
assert (actual_target, actual_version) == (node_target, node_version), (
f"{stage} changed the browser runtime: {node_target} ({node_version}) -> {actual_target} ({actual_version})"
)
url = "https://example.com"
port = get_free_port()
env = cli_env(
@@ -992,6 +1003,7 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
assert install_result.returncode == 0, (
f"archivebox install archivewebpage failed:\nSTDOUT:\n{install_result.stdout}\nSTDERR:\n{install_result.stderr}"
)
assert_browser_runtime_unchanged("archivebox install archivewebpage")
process = start_daemon_server(
initialized_archive,
@@ -1001,6 +1013,7 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
log_name="archivewebpage_server.log",
)
get_http_response(port, host=f"archivebox.localhost:{port}", path="/")
assert_browser_runtime_unchanged("archivebox server startup")
command_result = run_archivebox_cmd(
["add", "--depth=0", "--max-urls=1", "--plugins=archivewebpage", url],
cwd=initialized_archive,
@@ -1008,6 +1021,7 @@ def test_archivewebpage_wacz_preview_serves_real_capture_frame(initialized_archi
timeout=120,
)
assert command_result.returncode == 0, f"archivebox add failed:\nSTDOUT:\n{command_result.stdout}\nSTDERR:\n{command_result.stderr}"
assert_browser_runtime_unchanged("archivebox add --plugins=archivewebpage")
capture = _get_archivewebpage_capture(initialized_archive, url)
snapshot_host = f"{get_snapshot_subdomain(capture['snapshot_id'])}.archivebox.localhost:{port}"
+90
View File
@@ -83,6 +83,96 @@ def test_foreground_shutdown_can_request_cooperative_shutdown_without_raising():
assert process.returncode == 130, (stdout, stderr)
def test_cooperative_shutdown_during_child_poll_keeps_child_reapable():
"""Interrupt real child polling at different points without stranding reaping.
Immediate KeyboardInterrupt stranded Popen's wait lock in real CI, leaving
a zombie child that wait() could not reap. Exercise the cooperative policy
used by the server with the same external signals and child cleanup.
"""
process = _start_signal_process(
r"""
import os
import signal
import subprocess
import sys
import traceback
from collections import Counter
from archivebox.core.shutdown_util import (
foreground_shutdown_signals,
raise_if_shutdown_requested,
wait_popen_and_kill_children,
)
print("READY", flush=True)
interruption_sites = Counter()
for attempt in range(512):
child = subprocess.Popen(
[sys.executable, "-c", "import time; time.sleep(60)"],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
interruption_site = None
try:
with foreground_shutdown_signals(first_signal_message=None, raise_on_first_signal=False) as state:
try:
os.write(sys.stdout.fileno(), b"POLLING\n")
while True:
raise_if_shutdown_requested()
assert child.poll() is None
except KeyboardInterrupt as interruption:
assert state.signal_name == "SIGINT"
frame = traceback.extract_tb(interruption.__traceback__)[-1]
assert frame.name == "raise_if_shutdown_requested", frame
interruption_site = f"{frame.name}:{frame.lineno}"
interruption_sites[interruption_site] += 1
# The child deliberately remains alive until this unchanged
# kill-and-reap path asks it to stop.
wait_popen_and_kill_children(child, [], timeout=0.001, kill_timeout=1.0)
assert child.returncode == -signal.SIGKILL, (attempt, child.returncode)
except BaseException as error:
error.add_note(f"Interrupted child polling iteration: {attempt}, at {interruption_site}")
raise
finally:
# Cleanup must also work if the assertion exposes a broken
# Popen wait; preserve that failure while reaping the real PID.
if child.returncode is None:
try:
os.kill(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
try:
os.waitpid(child.pid, 0)
except ChildProcessError:
pass
print(f"Interrupt locations: {dict(interruption_sites)}", file=sys.stderr, flush=True)
print("REAPED:512", flush=True)
""",
)
try:
# An external sender does not contend for the polling interpreter's
# GIL. A Timer thread can bias delivery toward waitpid's GIL release,
# missing other interruption points reached by real CLI Ctrl+C.
assert process.stdout is not None
sent = 0
while sent < 512 and process.stdout.readline() == "POLLING\n":
process.send_signal(signal.SIGINT)
sent += 1
stdout, stderr = process.communicate(timeout=30)
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=5)
assert process.returncode == 0, (stdout, stderr)
assert sent == 512, (sent, stdout, stderr)
assert stdout == "REAPED:512\n", (stdout, stderr)
print(stderr)
def test_daemon_runner_signal_exit_is_unexpected_for_supervisor():
process = _start_signal_process(
"""
@@ -311,6 +311,38 @@ def test_responses_card_uses_relative_preview_urls_in_static_export(snapshot):
assert "localhost" not in html
@pytest.mark.django_db(transaction=True)
@pytest.mark.parametrize("status", ["queued", "started", "paused"])
def test_empty_snapshot_keeps_live_progress_until_files_are_opened(snapshot, live_server, status):
from urllib.parse import urlsplit
from playwright.sync_api import sync_playwright
from archivebox.machine.models import Machine
port = urlsplit(live_server.url).port
machine = Machine.current()
machine.config = {**machine.config, "BASE_URL": f"http://archivebox.localhost:{port}"}
machine.save(update_fields=["config"])
snapshot.permissions = "public"
snapshot.status = status
snapshot.save(update_fields=["permissions", "status"])
Path(snapshot.output_dir).mkdir(parents=True, exist_ok=True)
with sync_playwright() as playwright:
browser = playwright.chromium.launch(args=["--host-resolver-rules=MAP *.archivebox.localhost 127.0.0.1"])
page = browser.new_page()
page.goto(f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}")
assert page.locator("#progress-monitor").is_visible()
assert page.locator("#progress-monitor .progress-content").is_visible()
assert page.locator("#snapshot-empty-state").is_visible()
assert page.locator("#main-frame").count() == 0
page.get_by_role("button", name=re.compile("^Other files,")).click()
page.locator("#main-frame").wait_for()
assert page.locator("#progress-monitor").count() == 0
browser.close()
@pytest.mark.django_db(transaction=True)
def test_opening_files_from_queued_snapshot_stops_detached_progress_monitor(snapshot, live_server):
from urllib.parse import urlsplit
File diff suppressed because it is too large. Load diff
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python3
"""Takeover utility tests and live command handoff flows."""
import os
import json
import signal
import subprocess
import sys
import pty
import select
import termios
import time
from pathlib import Path
import pytest
import psutil
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.conftest import (
assert_no_processes_for_data_dir,
get_free_port,
kill_processes_for_data_dir,
cli_env,
pid_is_alive,
run_archivebox_cmd,
start_archivebox_server,
stop_archivebox_process,
get_http_response,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
pytestmark = pytest.mark.django_db(transaction=True)
def _resolve_sonic_env(env: dict[str, str]) -> dict[str, str]:
from abx_plugins import get_plugins_dir
config = Path(get_plugins_dir()) / "search_backend_sonic" / "config.json"
result = subprocess.run(
[
str(Path(sys.executable).with_name("abxpkg")),
"env",
"--install",
"--json",
f"--lib={env['ABXPKG_LIB_DIR']}",
f"--deps-from={config}:required_binaries",
],
capture_output=True,
text=True,
env=env,
)
assert result.returncode == 0, result.stderr or result.stdout
payload = {str(key): str(value) for key, value in json.loads(result.stdout).items()}
assert Path(payload["SONIC_BINARY"]).is_file()
return payload
def _archive_pages_for_sqlite_reindexing(data_dir: Path, env: dict[str, str], root_url: str) -> None:
add_env = dict(env)
add_env["SEARCH_BACKEND_ENGINE"] = "ripgrep"
_cmd_result = run_archivebox_cmd(
[
"add",
"--depth=2",
"--max-urls=20",
"--crawl-max-size=50mb",
"--plugins=wget,parse_html_urls",
root_url,
],
cwd=data_dir,
env=add_env,
timeout=240,
)
stdout, stderr, returncode = _cmd_result.stdout, _cmd_result.stderr, _cmd_result.returncode
assert returncode == 0, stderr or stdout
with use_archivebox_db(data_dir):
assert Snapshot.objects.filter(status=Snapshot.StatusChoices.SEALED).count() >= 1
assert not ArchiveResult.objects.filter(plugin="search_backend_sqlite").exists()
def _start_archivebox_shell(tmp_path: Path):
process = run_archivebox_cmd(
["manage", "shell"],
cwd=tmp_path,
env=cli_env(live=True),
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
capture_output=False,
start_new_session=True,
wait=False,
)
assert process.pid is not None
assert process.stdin is not None
assert pid_is_alive(process.pid)
return process
def _stop_archivebox_shells(processes) -> None:
for process in processes:
if process.stdin is not None and not process.stdin.closed:
process.stdin.close()
for process in processes:
process.wait(timeout=20)
assert not pid_is_alive(process.pid)
@pytest.mark.parametrize("interrupt_before_takeover", [False, True], ids=["running", "prompt"])
@pytest.mark.parametrize("after_takeover", ["resume", "cancel-standby"])
def test_interactive_add_takeover(initialized_archive, interrupt_before_takeover, after_takeover):
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 160))
output = bytearray()
add = None
server = None
port = get_free_port()
env = cli_env(
SEARCH_BACKEND_ENGINE="ripgrep",
SEARCH_BACKEND_SONIC_ENABLED="False",
CHROME_DELAY_AFTER_LOAD="60",
CHROME_TIMEOUT="120",
CHROME_HEADLESS="True",
)
installed = run_archivebox_cmd(["install", "chrome"], cwd=initialized_archive, env=env, timeout=600)
assert installed.returncode == 0, installed.stderr or installed.stdout
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
raise AssertionError(output.decode(errors="replace"))
def active_hook_for(parent_pid):
with use_archivebox_db(initialized_archive):
for hook in Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
status="running",
):
if hook.is_running and parent_pid in {parent.pid for parent in psutil.Process(hook.pid).parents()}:
return hook
return None
try:
add = run_archivebox_cmd(
["add", "--plugins=chrome", "https://example.com"],
cwd=initialized_archive,
env=env,
stdin=slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
read_until(lambda: active_hook_for(add.pid) is not None)
old_hook = active_hook_for(add.pid)
assert old_hook is not None
if interrupt_before_takeover:
add.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
assert not pid_is_alive(old_hook.pid)
server = start_archivebox_server(initialized_archive, port=port, log_name="interrupt-takeover-server.log", env=env)
read_until(lambda: b"A newer archivebox process took over" in output)
assert add.poll() is None, output.decode(errors="replace")
assert termios.tcgetattr(slave)[3] & termios.ICANON
assert not pid_is_alive(old_hook.pid)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
if after_takeover == "cancel-standby":
add.send_signal(signal.SIGINT)
read_until(lambda: add.poll() is not None)
assert add.returncode == 130, output.decode(errors="replace")
assert pid_is_alive(server.pid)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
else:
read_until(lambda: active_hook_for(server.pid) is not None)
stop_archivebox_process(server, signal.SIGTERM)
server = None
read_until(lambda: active_hook_for(add.pid) is not None)
resumed_hook = active_hook_for(add.pid)
assert resumed_hook is not None and resumed_hook.pid != old_hook.pid
prompt_offset = len(output)
add.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output[prompt_offset:])
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
os.write(master, b"\r")
read_until(lambda: add.poll() is not None)
assert add.returncode == 0, output.decode(errors="replace")
with use_archivebox_db(initialized_archive):
assert Crawl.objects.get().status == Crawl.StatusChoices.SEALED
assert b"Traceback" not in output
finally:
for process in (server, add):
if process is not None and process.poll() is None:
stop_archivebox_process(process, signal.SIGTERM)
os.close(slave)
os.close(master)
kill_processes_for_data_dir(initialized_archive)
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
+369
View File
@@ -0,0 +1,369 @@
#!/usr/bin/env python3
"""Takeover utility tests and live command handoff flows."""
import os
import re
import shlex
import shutil
import signal
import subprocess
import sys
import pty
import select
import termios
import time
from pathlib import Path
import pytest
from archivebox.machine.models import Process
from archivebox.tests.conftest import (
assert_no_processes_for_data_dir,
get_free_port,
kill_processes_for_data_dir,
cli_env,
pid_is_alive,
run_archivebox_cmd,
start_archivebox_server,
stop_archivebox_process,
get_http_response,
wait_for_snapshot_capture,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
from .test_takeover_util_1 import (
pytestmark as pytestmark,
_resolve_sonic_env as _resolve_sonic_env,
_archive_pages_for_sqlite_reindexing as _archive_pages_for_sqlite_reindexing,
_start_archivebox_shell as _start_archivebox_shell,
_stop_archivebox_shells as _stop_archivebox_shells,
)
@pytest.mark.parametrize("force_abort", [None, "typed", "signal"], ids=["graceful", "force-after-choice", "force-after-ctrl-c"])
def test_interactive_add_borrowing_server_supervisor_can_abort(initialized_archive, force_abort):
"""A server-owned supervisor must not make an interactive add worker noninteractive."""
master, slave = pty.openpty()
termios.tcsetwinsize(slave, (40, 160))
output = bytearray()
server = None
add = None
port = get_free_port()
env = cli_env(
SEARCH_BACKEND_ENGINE="ripgrep",
SEARCH_BACKEND_SONIC_ENABLED="False",
CHROME_DELAY_AFTER_LOAD="60",
CHROME_TIMEOUT="120",
CHROME_HEADLESS="True",
)
installed = run_archivebox_cmd(["install", "chrome"], cwd=initialized_archive, env=env, timeout=600)
assert installed.returncode == 0, installed.stderr or installed.stdout
def read_until(predicate, timeout=45):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if select.select([master], [], [], 0.1)[0]:
output.extend(os.read(master, 65536))
if predicate():
return
raise AssertionError(output.decode(errors="replace"))
def active_hook():
with use_archivebox_db(initialized_archive):
for hook in Process.objects.filter(
process_type=Process.TypeChoices.HOOK,
cmd__0__endswith="on_Snapshot__30_chrome_navigate.js",
status="running",
):
if hook.is_running:
return hook
return None
try:
server = start_archivebox_server(initialized_archive, port=port, log_name="add-borrows-server.log", env=env)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
add = run_archivebox_cmd(
["add", "--plugins=chrome", "https://example.com"],
cwd=initialized_archive,
env=env,
stdin=slave,
stdout=slave,
stderr=slave,
wait=False,
start_new_session=True,
)
read_until(lambda: active_hook() is not None)
old_hook = active_hook()
assert old_hook is not None
add.send_signal(signal.SIGINT)
read_until(lambda: b"Choice [skip]:" in output)
read_until(lambda: not termios.tcgetattr(slave)[3] & termios.ICANON)
assert add.poll() is None, output.decode(errors="replace")
assert not pid_is_alive(old_hook.pid)
if force_abort:
# Both abort choices must become forceable before the worker has
# time to write its marker or finish the normal hook grace phase.
if force_abort == "typed":
os.write(master, b"a")
else:
add.send_signal(signal.SIGINT)
read_until(lambda: termios.tcgetattr(slave)[3] & termios.ICANON)
forced_at = time.monotonic()
add.send_signal(signal.SIGINT)
else:
add.send_signal(signal.SIGINT)
read_until(lambda: add.poll() is not None)
if force_abort:
assert time.monotonic() - forced_at < 2.0, output.decode(errors="replace")
assert b"Forcing aborted crawl to exit now" in output
assert add.returncode == 130, output.decode(errors="replace")
assert pid_is_alive(server.pid)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
assert b"Traceback" not in output
finally:
for process in (add, server):
if process is not None and process.poll() is None:
stop_archivebox_process(process, signal.SIGTERM)
os.close(slave)
os.close(master)
kill_processes_for_data_dir(initialized_archive)
assert_no_processes_for_data_dir(initialized_archive, timeout=12)
def test_explicit_sonic_binary_is_used_by_real_search_worker(initialized_archive):
sonic_port = get_free_port()
lib_dir = initialized_archive / "lib"
env = cli_env(
live=True,
ABXPKG_LIB_DIR=str(lib_dir),
PLUGINS="search_backend_sonic",
SEARCH_BACKEND_ENGINE="sonic",
SEARCH_BACKEND_SONIC_PORT=str(sonic_port),
)
env.update(_resolve_sonic_env(env))
env["PATH"] = os.pathsep.join((str(Path(sys.executable).parent), env["PATH"]))
installed_binary = Path(env["SONIC_BINARY"]).resolve(strict=True)
managed_sonic_dir = lib_dir / "bash" / "bin"
managed_sonic_dir.mkdir(parents=True, exist_ok=True)
managed_binary = managed_sonic_dir / "sonic"
if installed_binary != managed_binary:
shutil.copy2(installed_binary, managed_binary)
config_result = run_archivebox_cmd(
["config", "--set", f"SONIC_BINARY={managed_binary}"],
cwd=initialized_archive,
env=env,
timeout=60,
)
assert config_result.returncode == 0, config_result.stderr or config_result.stdout
pinned_sonic_dir = initialized_archive / "pinned-sonic"
pinned_sonic_dir.mkdir()
sonic_binary = pinned_sonic_dir / "sonic"
shutil.copy2(installed_binary, sonic_binary)
env["SONIC_BINARY"] = str(sonic_binary)
result = run_archivebox_cmd(
["list", "--search=contents", "--csv=url", "not-indexed"],
cwd=initialized_archive,
env=env,
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
config_match = re.search(r"Using supervisord config file: (\S+)", result.stderr)
assert config_match, result.stderr
worker_config = Path(config_match.group(1)).parent / "workers" / "worker_sonic.conf"
worker_text = worker_config.read_text(encoding="utf-8")
command = next(line.partition("=")[2] for line in worker_text.splitlines() if line.startswith("command="))
assert shlex.split(command)[0] == str(sonic_binary), worker_text
from archivebox.workers.supervisord_util import _sonic_worker_bind_target
assert _sonic_worker_bind_target({"name": "worker_sonic", "command": command}) == ("127.0.0.1", sonic_port)
renamed_binary = initialized_archive / "renamed-sonic-binary"
shutil.copy2(sonic_binary, renamed_binary)
renamed_command = shlex.join([str(renamed_binary), *shlex.split(command)[1:]])
assert _sonic_worker_bind_target({"name": "worker_sonic", "command": renamed_command}) == ("127.0.0.1", sonic_port)
relative_binary = pinned_sonic_dir / "sonic-alt"
shutil.copy2(sonic_binary, relative_binary)
relative_env = {
**env,
"SONIC_BINARY": relative_binary.name,
"PATH": os.pathsep.join((str(pinned_sonic_dir), env["PATH"])),
"SEARCH_BACKEND_SONIC_PORT": str(get_free_port()),
}
relative_result = run_archivebox_cmd(
["list", "--search=contents", "--csv=url", "not-indexed"],
cwd=initialized_archive,
env=relative_env,
timeout=60,
)
assert relative_result.returncode == 0, relative_result.stderr or relative_result.stdout
relative_worker_text = worker_config.read_text(encoding="utf-8")
relative_command = next(line.partition("=")[2] for line in relative_worker_text.splitlines() if line.startswith("command="))
assert Path(shlex.split(relative_command)[0]).name == relative_binary.name, relative_worker_text
@pytest.mark.timeout(300)
def test_behavior_foreground_add_keeps_existing_server_http_visible(tmp_path, initialized_archive, recursive_test_site):
port = get_free_port()
env = cli_env(live=True, server=True, port=port, SEARCH_BACKEND_ENGINE="ripgrep")
server = None
try:
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-add.log", env=env)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
add = run_archivebox_cmd(
[
"add",
"--depth=2",
"--max-urls=10",
"--plugins=wget,parse_html_urls",
recursive_test_site["root_url"],
],
cwd=tmp_path,
env=env,
timeout=180,
)
assert add.returncode == 0, add.stderr or add.stdout
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=120)
assert "Root" in captured_text
finally:
if server is not None:
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(300)
def test_behavior_background_add_returns_and_server_archives_visible_url(tmp_path, initialized_archive, recursive_test_site):
port = get_free_port()
env = cli_env(live=True, server=True, port=port, SEARCH_BACKEND_ENGINE="ripgrep")
server = None
try:
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-bg-add.log", env=env)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
add = run_archivebox_cmd(
[
"add",
"--bg",
"--depth=0",
"--plugins=wget",
recursive_test_site["root_url"],
],
cwd=tmp_path,
env=env,
timeout=60,
)
assert add.returncode == 0, add.stderr or add.stdout
assert "background runner will process" in add.stdout
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
assert "Root" in captured_text
finally:
if server is not None:
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
def test_foreground_runner_owner_prefers_newer_update_over_server(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner
procs: list[subprocess.Popen[str]] = []
try:
for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE):
proc = _start_archivebox_shell(tmp_path)
procs.append(proc)
Process.objects.create(
machine=Machine.current(),
process_type=process_type,
worker_type=process_type,
pwd=str(tmp_path),
cmd=[],
pid=proc.pid,
status=Process.StatusChoices.RUNNING,
)
runtime_owner = runtime_stack_owner(data_dir=tmp_path)
runner_owner = foreground_runner_owner(data_dir=tmp_path)
assert runtime_owner is not None
assert runtime_owner.process_type == Process.TypeChoices.SERVER
assert runner_owner is not None
assert runner_owner.process_type == Process.TypeChoices.UPDATE
assert runner_owner.pid == procs[-1].pid
finally:
_stop_archivebox_shells(procs)
def test_runtime_stack_owner_ignores_supervised_orphan_runner(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner
procs: list[subprocess.Popen[str]] = []
try:
for _ in range(2):
proc = _start_archivebox_shell(tmp_path)
procs.append(proc)
supervisor_row = Process.objects.create(
machine=Machine.current(),
process_type=Process.TypeChoices.SUPERVISORD,
worker_type="supervisord",
pwd=str(tmp_path),
cmd=[],
pid=procs[0].pid,
status=Process.StatusChoices.RUNNING,
)
Process.objects.create(
machine=Machine.current(),
parent=supervisor_row,
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
pwd=str(tmp_path),
cmd=[],
pid=procs[1].pid,
status=Process.StatusChoices.RUNNING,
)
assert runtime_stack_owner(data_dir=tmp_path) is None
finally:
_stop_archivebox_shells(procs)
def test_foreign_machine_runner_only_warns(tmp_path, initialized_archive, capsys):
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, live_runner_processes
from archivebox.machine.models import Machine
foreign_machine = Machine.objects.create(
guid="foreign-machine",
hostname="foreign-host",
hw_manufacturer="Test",
hw_product="Test",
hw_uuid="foreign-hardware",
os_arch="x86_64",
os_family="linux",
os_platform="linux",
os_release="test",
os_kernel="test",
)
foreign_runner = Process.objects.create(
machine=foreign_machine,
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
pwd=str(tmp_path),
pid=1,
status=Process.StatusChoices.RUNNING,
)
assert live_runner_processes(data_dir=tmp_path) == []
foreign_runner.refresh_from_db()
assert foreign_runner.status == Process.StatusChoices.RUNNING
assert "Multiple orchestrators sharing a single collection is not officially supported" in capsys.readouterr().err
+400
View File
@@ -0,0 +1,400 @@
#!/usr/bin/env python3
"""Takeover utility tests and live command handoff flows."""
import signal
import subprocess
import pytest
import psutil
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
from archivebox.machine.models import Process
from archivebox.tests.conftest import (
assert_no_processes_for_data_dir,
get_free_port,
kill_processes_for_data_dir,
cli_env,
pid_is_alive,
run_archivebox_cmd,
start_archivebox_server,
stop_archivebox_process,
supervisor_pid_from_log,
get_http_response,
wait_for_log_count,
wait_for_pid_to_disappear,
wait_for_snapshot_capture,
worker_pid_from_log,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
from .test_takeover_util_1 import (
pytestmark as pytestmark,
_resolve_sonic_env as _resolve_sonic_env,
_archive_pages_for_sqlite_reindexing as _archive_pages_for_sqlite_reindexing,
_start_archivebox_shell as _start_archivebox_shell,
_stop_archivebox_shells as _stop_archivebox_shells,
)
@pytest.mark.timeout(300)
def test_behavior_daemonized_server_restarts_cleanly_after_forced_stop(tmp_path, initialized_archive):
port = get_free_port()
env = cli_env(live=True, server=True, port=port)
try:
first = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True)
assert first.returncode == 0, first.stderr or first.stdout
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
second = start_archivebox_server(tmp_path, port=port, env=env, daemonize=True)
assert second.returncode == 0, second.stderr or second.stdout
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
finally:
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(240)
def test_live_second_server_takes_over_existing_server_process(tmp_path, initialized_archive):
env = cli_env(live=True)
port = get_free_port()
first = None
second = None
try:
first = start_archivebox_server(tmp_path, port=port, log_name="server-first.log", env=env)
first_log = first.log_path
second = start_archivebox_server(tmp_path, port=port, log_name="server-second.log", env=env)
second_log = second.log_path
assert pid_is_alive(first.pid)
first_text = first_log.read_text(encoding="utf-8", errors="replace")
second_text = second_log.read_text(encoding="utf-8", errors="replace")
assert "A newer archivebox process took over the orchestrator, server" in first_text
assert "Starting orchestrator, server" in second_text
result = run_archivebox_cmd(
["status"],
cwd=tmp_path,
env=env,
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
first_resumes = first_log.read_text(encoding="utf-8", errors="replace").count("Other newer archivebox process")
stop_archivebox_process(second, signal.SIGTERM)
second = None
wait_for_log_count(first_log, "Other newer archivebox process", first_resumes + 1, timeout=35)
assert pid_is_alive(first.pid)
finally:
if second is not None:
stop_archivebox_process(second, signal.SIGTERM)
if first is not None:
stop_archivebox_process(first, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(360)
def test_live_server_keeps_http_runtime_while_update_runs_real_sqlite_indexer(tmp_path, initialized_archive, recursive_test_site):
env = cli_env(
live=True,
PLUGINS="wget,parse_html_urls,search_backend_sqlite,search_backend_sonic",
SEARCH_BACKEND_ENGINE="sqlite",
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
)
env.update(_resolve_sonic_env(env))
_archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"])
port = get_free_port()
server = None
try:
server = start_archivebox_server(tmp_path, port=port, log_name="server-real-sqlite-update.log", env=env)
server_log = server.log_path
supervisor_pid_before = supervisor_pid_from_log(server_log)
daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne")
runner_pid_before = worker_pid_from_log(server_log, "worker_runner")
sonic_pid_before = worker_pid_from_log(server_log, "worker_sonic")
supervisord_log = tmp_path / "logs" / "supervisord.log"
runner_spawn_text = "spawned: 'worker_runner' with pid"
runner_spawn_count = supervisord_log.read_text(encoding="utf-8", errors="replace").count(runner_spawn_text)
result = run_archivebox_cmd(
["update", "--index-only", "--batch-size=1"],
cwd=tmp_path,
env=env,
timeout=180,
)
assert result.returncode == 0, result.stderr or result.stdout
assert pid_is_alive(server.pid)
assert supervisor_pid_from_log(server_log) == supervisor_pid_before
assert pid_is_alive(daphne_pid_before)
assert pid_is_alive(sonic_pid_before)
assert worker_pid_from_log(server_log, "worker_daphne") == daphne_pid_before
assert worker_pid_from_log(server_log, "worker_sonic") == sonic_pid_before
assert "A newer archivebox process took over the orchestrator, server" not in server_log.read_text(
encoding="utf-8",
errors="replace",
)
supervisord_text = supervisord_log.read_text(encoding="utf-8", errors="replace")
assert supervisord_text.count(runner_spawn_text) == runner_spawn_count
assert worker_pid_from_log(server_log, "worker_runner") == runner_pid_before
assert pid_is_alive(runner_pid_before)
with use_archivebox_db(tmp_path):
indexed_results = list(
ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True),
)
assert indexed_results
assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results)
stop_archivebox_process(server, signal.SIGTERM)
server = None
wait_for_pid_to_disappear(daphne_pid_before, timeout=20)
wait_for_pid_to_disappear(sonic_pid_before, timeout=20)
wait_for_pid_to_disappear(runner_pid_before, timeout=20)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
finally:
if server is not None:
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(420)
def test_live_repeated_server_startups_take_over_cleanly(tmp_path, initialized_archive):
env = cli_env(live=True)
port = get_free_port()
servers: list[subprocess.Popen[str]] = []
server_pids: list[int] = []
daphne_pids: list[int] = []
runner_pids: list[int] = []
try:
for index in range(5):
server = start_archivebox_server(tmp_path, port=port, log_name=f"server-chaos-{index}.log", env=env)
log_path = server.log_path
servers.append(server)
server_pids.append(server.pid)
daphne_pids.append(worker_pid_from_log(log_path, "worker_daphne"))
runner_pids.append(worker_pid_from_log(log_path, "worker_runner"))
if index > 0:
previous_server = servers[index - 1]
previous_log = (tmp_path / f"server-chaos-{index - 1}.log").read_text(encoding="utf-8", errors="replace")
current_log = log_path.read_text(encoding="utf-8", errors="replace")
assert pid_is_alive(previous_server.pid)
assert pid_is_alive(server_pids[index - 1])
assert "A newer archivebox process took over the orchestrator, server" in previous_log
assert "Starting orchestrator, server" in current_log
wait_for_pid_to_disappear(daphne_pids[index - 1], timeout=15)
wait_for_pid_to_disappear(runner_pids[index - 1], timeout=15)
result = run_archivebox_cmd(
["status"],
cwd=tmp_path,
env=env,
timeout=60,
)
assert result.returncode == 0, result.stderr or result.stdout
assert pid_is_alive(servers[-1].pid)
assert all(pid_is_alive(server.pid) for server in servers)
# Inspect the actual server worker rather than requiring a host lsof
# installation. Previous workers must be gone, and the newest worker
# must own exactly one listening socket on our assigned port.
assert all(not pid_is_alive(pid) for pid in daphne_pids[:-1])
listeners = [
connection
for connection in psutil.Process(daphne_pids[-1]).net_connections(kind="tcp")
if connection.status == psutil.CONN_LISTEN and connection.laddr.port == port
]
assert len(listeners) == 1, listeners
previous_log_path = tmp_path / "server-chaos-3.log"
previous_takeovers = previous_log_path.read_text(encoding="utf-8", errors="replace").count(
"Other newer archivebox process",
)
stop_archivebox_process(servers[-1], signal.SIGTERM)
wait_for_log_count(previous_log_path, "Other newer archivebox process", previous_takeovers + 1, timeout=35)
assert pid_is_alive(servers[3].pid)
finally:
for server in reversed(servers):
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(420)
def test_live_background_add_survives_server_exit_and_foreground_run_reclaims(tmp_path, initialized_archive, recursive_test_site):
env = cli_env(live=True, SEARCH_BACKEND_ENGINE="ripgrep")
port = get_free_port()
server = None
server2 = None
try:
server = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-1.log", env=env)
server_log = server.log_path
supervisor_pid_before = supervisor_pid_from_log(server_log)
result = run_archivebox_cmd(
["update", "--index-only", "--batch-size=10"],
cwd=tmp_path,
env=env,
timeout=90,
)
assert result.returncode == 0, result.stderr or result.stdout
assert pid_is_alive(server.pid)
assert pid_is_alive(supervisor_pid_before)
assert supervisor_pid_from_log(server_log) == supervisor_pid_before
add = run_archivebox_cmd(
[
"add",
"--bg",
"--depth=1",
"--max-urls=20",
"--crawl-max-size=50mb",
"--plugins=wget,parse_html_urls",
recursive_test_site["root_url"],
],
cwd=tmp_path,
env=env,
timeout=60,
)
assert add.returncode == 0, add.stderr or add.stdout
assert "background runner will process" in add.stdout
stop_archivebox_process(server, signal.SIGTERM)
server = None
assert "Got SIGTERM" in server_log.read_text(encoding="utf-8", errors="replace")
assert_no_processes_for_data_dir(tmp_path, timeout=12)
with use_archivebox_db(tmp_path):
crawl_id = str(Crawl.objects.get().id)
run = run_archivebox_cmd(
["run", f"--crawl-id={crawl_id}"],
cwd=tmp_path,
env=env,
timeout=180,
)
assert run.returncode == 0, run.stderr or run.stdout
server2 = start_archivebox_server(tmp_path, port=port, log_name="server-add-owner-2.log", env=env)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
captured_text = wait_for_snapshot_capture(tmp_path, recursive_test_site["root_url"], timeout=180)
assert "Root" in captured_text
stop_archivebox_process(server2, signal.SIGTERM)
server2 = None
with use_archivebox_db(tmp_path):
crawls = list(Crawl.objects.order_by("created_at").values_list("status", "retry_at"))
snapshots = list(Snapshot.objects.order_by("created_at").values_list("url", "status", "retry_at"))
bad_results = list(
ArchiveResult.objects.filter(
status__in=[
ArchiveResult.StatusChoices.FAILED,
ArchiveResult.StatusChoices.SKIPPED,
],
).values_list("plugin", "status", "output_str"),
)
assert crawls
assert snapshots
assert all(status == Crawl.StatusChoices.SEALED for status, _retry_at in crawls)
assert all(status == Snapshot.StatusChoices.SEALED for _url, status, _retry_at in snapshots)
assert not bad_results
finally:
for proc in (server, server2):
if proc is not None:
stop_archivebox_process(proc, signal.SIGTERM, timeout=10)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
# Utility-level takeover selection tests.
def test_runtime_stack_owner_prefers_newer_server_over_older_update(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import runtime_stack_owner
procs: list[subprocess.Popen[str]] = []
try:
for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER):
proc = _start_archivebox_shell(tmp_path)
procs.append(proc)
Process.objects.create(
machine=Machine.current(),
process_type=process_type,
worker_type=process_type,
pwd=str(tmp_path),
cmd=[],
pid=proc.pid,
status=Process.StatusChoices.RUNNING,
)
owner = runtime_stack_owner(data_dir=tmp_path)
assert owner is not None
assert owner.process_type == Process.TypeChoices.SERVER
assert owner.pid == procs[-1].pid
finally:
_stop_archivebox_shells(procs)
def test_foreground_runner_owner_prefers_newer_server_over_update(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import foreground_runner_owner, runtime_stack_owner
procs: list[subprocess.Popen[str]] = []
try:
for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER):
proc = _start_archivebox_shell(tmp_path)
procs.append(proc)
Process.objects.create(
machine=Machine.current(),
process_type=process_type,
worker_type=process_type,
pwd=str(tmp_path),
cmd=[],
pid=proc.pid,
status=Process.StatusChoices.RUNNING,
)
runtime_owner = runtime_stack_owner(data_dir=tmp_path)
runner_owner = foreground_runner_owner(data_dir=tmp_path)
assert runtime_owner is not None
assert runtime_owner.process_type == Process.TypeChoices.SERVER
assert runner_owner is not None
assert runner_owner.process_type == Process.TypeChoices.SERVER
assert runner_owner.pid == procs[-1].pid
finally:
_stop_archivebox_shells(procs)
def test_runtime_stack_owner_allows_top_level_runner_when_no_parent_command_exists(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner
proc = _start_archivebox_shell(tmp_path)
try:
runner_row = Process.objects.create(
machine=Machine.current(),
process_type=Process.TypeChoices.ORCHESTRATOR,
worker_type=RUNNER_ACTIVE_WORKER_TYPE,
pwd=str(tmp_path),
cmd=[],
pid=proc.pid,
status=Process.StatusChoices.RUNNING,
)
owner = runtime_stack_owner(data_dir=tmp_path)
assert owner is not None
assert owner.id == runner_row.id
finally:
_stop_archivebox_shells([proc])
+377
View File
@@ -0,0 +1,377 @@
#!/usr/bin/env python3
"""Takeover utility tests and live command handoff flows."""
import os
import signal
import subprocess
import pytest
import psutil
from archivebox.core.models import ArchiveResult
from archivebox.machine.models import Process
from archivebox.tests.conftest import (
assert_no_processes_for_data_dir,
get_free_port,
kill_processes_for_data_dir,
cli_env,
pid_is_alive,
run_archivebox_cmd,
start_archivebox_server,
stop_archivebox_process,
supervisor_pid_from_log,
get_http_response,
wait_for_log,
wait_for_log_pattern,
wait_for_pid_to_disappear,
wait_for_worker_pid_from_log,
worker_pid_from_log,
)
from archivebox.tests.test_orm_helpers import use_archivebox_db
from .test_takeover_util_1 import (
pytestmark as pytestmark,
_resolve_sonic_env as _resolve_sonic_env,
_archive_pages_for_sqlite_reindexing as _archive_pages_for_sqlite_reindexing,
_start_archivebox_shell as _start_archivebox_shell,
_stop_archivebox_shells as _stop_archivebox_shells,
)
def test_pid_is_alive_treats_unreaped_archivebox_cli_as_exited(tmp_path, initialized_archive):
proc = run_archivebox_cmd(
["version"],
cwd=tmp_path,
default_cli_env=True,
disable_extractors=True,
wait=False,
)
try:
os.waitid(os.P_PID, proc.pid, os.WEXITED | os.WNOWAIT)
assert psutil.Process(proc.pid).status() == psutil.STATUS_ZOMBIE
assert not pid_is_alive(proc.pid)
finally:
proc.wait(timeout=5)
@pytest.mark.timeout(360)
def test_behavior_update_index_only_keeps_server_http_and_search_visible(tmp_path, initialized_archive, recursive_test_site):
env = cli_env(
live=True,
PLUGINS="wget,parse_html_urls,search_backend_sqlite",
SEARCH_BACKEND_ENGINE="sqlite",
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
)
env.update(_resolve_sonic_env(env))
root_url = recursive_test_site["root_url"]
_archive_pages_for_sqlite_reindexing(tmp_path, env, root_url)
port = get_free_port()
server = None
try:
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-update.log", env=env)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
update = run_archivebox_cmd(
["update", "--index-only", "--batch-size=1"],
cwd=tmp_path,
env=env,
timeout=180,
)
assert update.returncode == 0, update.stderr or update.stdout
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
search = run_archivebox_cmd(
["list", "--search=contents", "--csv=url", "Root"],
cwd=tmp_path,
env=env,
timeout=60,
)
assert search.returncode == 0, search.stderr or search.stdout
assert root_url in search.stdout
finally:
if server is not None:
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(420)
def test_behavior_update_yields_to_server_then_finishes_visible_indexing(tmp_path, initialized_archive, recursive_test_site):
env = cli_env(
live=True,
PLUGINS="wget,parse_html_urls,search_backend_sqlite",
SEARCH_BACKEND_ENGINE="sqlite",
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
)
env.update(_resolve_sonic_env(env))
root_url = recursive_test_site["root_url"]
_archive_pages_for_sqlite_reindexing(tmp_path, env, root_url)
port = get_free_port()
update_proc = None
server = None
try:
update_log = tmp_path / "behavior-update-yields.log"
update_log_handle = update_log.open("w", encoding="utf-8")
update_proc = run_archivebox_cmd(
["update", "--index-only", "--batch-size=1"],
cwd=tmp_path,
env=env,
stdout=update_log_handle,
stderr=subprocess.STDOUT,
start_new_session=True,
wait=False,
)
update_log_handle.close()
wait_for_log(update_log, "[*] Reindexing", timeout=90)
server = start_archivebox_server(tmp_path, port=port, log_name="behavior-server-takes-update.log", env=env)
assert get_http_response(port, host=f"archivebox.localhost:{port}").status_code < 500
wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90)
stop_archivebox_process(server, signal.SIGTERM)
server = None
update_proc.wait(timeout=180)
update_text = update_log.read_text(encoding="utf-8", errors="replace")
assert update_proc.returncode == 0, update_text
search = run_archivebox_cmd(
["list", "--search=contents", "--csv=url", "Root"],
cwd=tmp_path,
env=env,
timeout=60,
)
assert search.returncode == 0, search.stderr or search.stdout
assert root_url in search.stdout
finally:
if update_proc is not None:
stop_archivebox_process(update_proc, signal.SIGTERM)
if server is not None:
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(180)
def test_live_update_index_only_does_not_take_over_server_runtime(tmp_path, initialized_archive):
env = cli_env(live=True)
port = get_free_port()
server = None
try:
server = start_archivebox_server(tmp_path, port=port, log_name="server-update-owner.log", env=env)
server_log = server.log_path
supervisor_pid_before = supervisor_pid_from_log(server_log)
daphne_pid_before = worker_pid_from_log(server_log, "worker_daphne")
result = run_archivebox_cmd(
["update", "--index-only", "--before=0"],
cwd=tmp_path,
env=env,
timeout=90,
)
assert result.returncode == 0, result.stderr or result.stdout
assert pid_is_alive(server.pid)
assert pid_is_alive(supervisor_pid_before)
assert pid_is_alive(daphne_pid_before)
assert supervisor_pid_from_log(server_log) == supervisor_pid_before
assert worker_pid_from_log(server_log, "worker_daphne") == daphne_pid_before
assert "A newer archivebox process took over the orchestrator, server" not in server_log.read_text(
encoding="utf-8",
errors="replace",
)
finally:
if server is not None:
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
@pytest.mark.timeout(420)
def test_live_update_yields_to_server_then_reclaims_real_sqlite_indexing(tmp_path, initialized_archive, recursive_test_site):
env = cli_env(
live=True,
PLUGINS="wget,parse_html_urls,search_backend_sqlite,search_backend_sonic",
SEARCH_BACKEND_ENGINE="sqlite",
SEARCH_BACKEND_SONIC_PORT=str(get_free_port()),
)
env.update(_resolve_sonic_env(env))
_archive_pages_for_sqlite_reindexing(tmp_path, env, recursive_test_site["root_url"])
port = get_free_port()
update_proc = None
server = None
try:
update_log = tmp_path / "update-real-sqlite-owner.log"
update_log_handle = update_log.open("w", encoding="utf-8")
update_proc = run_archivebox_cmd(
["update", "--index-only", "--batch-size=1"],
cwd=tmp_path,
env=env,
stdout=update_log_handle,
stderr=subprocess.STDOUT,
start_new_session=True,
wait=False,
)
update_log_handle.close()
wait_for_log(update_log, "[*] Reindexing", timeout=90)
update_supervisor_match = wait_for_log_pattern(update_log, r"Supervisord connected \(pid=(\d+)\)", timeout=90)
update_supervisor_pid_before = int(update_supervisor_match.group(1))
update_sonic_pid_before = wait_for_worker_pid_from_log(update_log, "worker_sonic", timeout=90)
assert pid_is_alive(update_supervisor_pid_before)
assert pid_is_alive(update_sonic_pid_before)
assert "worker_runner_update_" not in update_log.read_text(encoding="utf-8", errors="replace")
assert "worker_daphne" not in update_log.read_text(encoding="utf-8", errors="replace")
server = start_archivebox_server(tmp_path, port=port, log_name="server-takes-real-sqlite-update.log", env=env)
server_log = server.log_path
wait_for_log(update_log, "A newer archivebox process took over the orchestrator, sonic", timeout=90)
assert pid_is_alive(update_proc.pid)
assert pid_is_alive(server.pid)
server_text = server_log.read_text(encoding="utf-8", errors="replace")
# The older update process can yield orchestrator ownership just before
# the server logs its takeover, but sonic must always move to the server.
assert (
"Taking over orchestrator, sonic from older existing archivebox process" in server_text
or "Taking over sonic from older existing archivebox process" in server_text
)
assert "worker_daphne" in server_text
assert "worker_sonic" in server_text
server_daphne_pid = worker_pid_from_log(server_log, "worker_daphne")
server_runner_pid = worker_pid_from_log(server_log, "worker_runner")
server_sonic_pid = worker_pid_from_log(server_log, "worker_sonic")
assert pid_is_alive(server_daphne_pid)
assert pid_is_alive(server_runner_pid)
assert pid_is_alive(server_sonic_pid)
wait_for_pid_to_disappear(update_supervisor_pid_before, timeout=30)
wait_for_pid_to_disappear(update_sonic_pid_before, timeout=30)
stop_archivebox_process(server, signal.SIGTERM)
server = None
update_proc.wait(timeout=180)
update_text = update_log.read_text(encoding="utf-8", errors="replace")
assert update_proc.returncode == 0, update_text
wait_for_pid_to_disappear(server_daphne_pid, timeout=20)
wait_for_pid_to_disappear(server_runner_pid, timeout=20)
wait_for_pid_to_disappear(server_sonic_pid, timeout=20)
with use_archivebox_db(tmp_path):
indexed_results = list(
ArchiveResult.objects.filter(plugin="search_backend_sqlite").values_list("status", flat=True),
)
assert indexed_results
assert all(status in ArchiveResult.FINAL_STATES for status in indexed_results)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
finally:
if update_proc is not None:
stop_archivebox_process(update_proc, signal.SIGTERM)
if server is not None:
stop_archivebox_process(server, signal.SIGTERM)
kill_processes_for_data_dir(tmp_path)
assert_no_processes_for_data_dir(tmp_path, timeout=12)
def test_runtime_stack_owner_keeps_server_over_newer_update(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import runtime_stack_owner
procs: list[subprocess.Popen[str]] = []
try:
for process_type in (Process.TypeChoices.SERVER, Process.TypeChoices.UPDATE):
proc = _start_archivebox_shell(tmp_path)
procs.append(proc)
Process.objects.create(
machine=Machine.current(),
process_type=process_type,
worker_type=process_type,
pwd=str(tmp_path),
cmd=[],
pid=proc.pid,
status=Process.StatusChoices.RUNNING,
)
owner = runtime_stack_owner(data_dir=tmp_path)
assert owner is not None
assert owner.process_type == Process.TypeChoices.SERVER
assert owner.pid == procs[0].pid
finally:
_stop_archivebox_shells(procs)
def test_runtime_stack_owner_keeps_server_over_newer_supervised_runner(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import RUNNER_ACTIVE_WORKER_TYPE, runtime_stack_owner
procs: list[subprocess.Popen[str]] = []
try:
for process_type, worker_type in (
(Process.TypeChoices.SERVER, ""),
(Process.TypeChoices.ORCHESTRATOR, RUNNER_ACTIVE_WORKER_TYPE),
):
proc = _start_archivebox_shell(tmp_path)
procs.append(proc)
Process.objects.create(
machine=Machine.current(),
process_type=process_type,
worker_type=worker_type,
pwd=str(tmp_path),
cmd=[],
pid=proc.pid,
status=Process.StatusChoices.RUNNING,
)
owner = runtime_stack_owner(data_dir=tmp_path)
assert owner is not None
assert owner.process_type == Process.TypeChoices.SERVER
assert owner.pid == procs[0].pid
finally:
_stop_archivebox_shells(procs)
def test_runtime_stack_owner_reaps_dead_server_without_promoting_update(tmp_path, initialized_archive):
from archivebox.machine.models import Machine
from archivebox.core.takeover_util import runtime_stack_owner
procs: list[subprocess.Popen[str]] = []
older_row = None
newer_row = None
try:
for process_type in (Process.TypeChoices.UPDATE, Process.TypeChoices.SERVER):
proc = _start_archivebox_shell(tmp_path)
procs.append(proc)
row = Process.objects.create(
machine=Machine.current(),
process_type=process_type,
worker_type=process_type,
pwd=str(tmp_path),
cmd=[],
pid=proc.pid,
status=Process.StatusChoices.RUNNING,
)
if process_type == Process.TypeChoices.UPDATE:
older_row = row
else:
newer_row = row
assert older_row is not None
assert newer_row is not None
os.kill(procs[-1].pid, signal.SIGTERM)
procs[-1].wait(timeout=20)
owner = runtime_stack_owner(data_dir=tmp_path)
assert owner is None
newer_row.refresh_from_db()
assert newer_row.status == Process.StatusChoices.EXITED
older_row.refresh_from_db()
assert older_row.status == Process.StatusChoices.RUNNING
finally:
_stop_archivebox_shells(procs)
+36 -61
View File
@@ -209,10 +209,13 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s
_stop_worker(tmp_path, "worker_runner")
assert _worker_state(tmp_path, "worker_runner") != "RUNNING"
session, csrf_token = _login_to_add_view(port)
session, _add_page, csrf_token = _login_to_add_view(port)
response = session.post(
f"http://admin.archivebox.localhost:{port}/add/",
headers={"Referer": f"http://admin.archivebox.localhost:{port}/add/"},
f"http://127.0.0.1:{port}/add/",
headers={
"Host": f"admin.archivebox.localhost:{port}",
"Referer": f"http://admin.archivebox.localhost:{port}/add/",
},
data={
"url": recursive_test_site["root_url"],
"depth": "0",
@@ -246,19 +249,26 @@ def test_add_view_restarts_stopped_supervisord_runner(tmp_path, recursive_test_s
stop_server(tmp_path)
def _login_to_add_view(port: int) -> tuple[requests.Session, str]:
def _login_to_add_view(port: int) -> tuple[requests.Session, requests.Response, str]:
# Browsers resolve *.localhost themselves; system DNS in stock containers
# need not. Loopback transport with the real Host/Referer preserves virtual
# host routing, session cookies and CSRF checks without depending on DNS.
session = requests.Session()
get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
login_page = session.get(
f"http://admin.archivebox.localhost:{port}/admin/login/",
f"http://127.0.0.1:{port}/admin/login/",
headers={"Host": f"admin.archivebox.localhost:{port}"},
timeout=10,
)
assert login_page.status_code == 200
csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
assert csrf_match, login_page.text[:500]
login_response = session.post(
f"http://admin.archivebox.localhost:{port}/admin/login/",
headers={"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/"},
f"http://127.0.0.1:{port}/admin/login/",
headers={
"Host": f"admin.archivebox.localhost:{port}",
"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/",
},
data={
"username": "apitestadmin",
"password": "testpass123",
@@ -270,12 +280,13 @@ def _login_to_add_view(port: int) -> tuple[requests.Session, str]:
)
assert login_response.status_code in (302, 303), login_response.text
add_page, csrf_token = _get_add_view(session, port, host=f"admin.archivebox.localhost:{port}")
return session, csrf_token
return session, add_page, csrf_token
def _get_add_view(session: requests.Session, port: int, *, host: str) -> tuple[requests.Response, str]:
add_page = session.get(
f"http://{host}/add/",
f"http://127.0.0.1:{port}/add/",
headers={"Host": host},
timeout=10,
)
assert add_page.status_code == 200
@@ -612,33 +623,14 @@ def test_add_view_post_creates_schedule_over_server(tmp_path, recursive_test_sit
try:
start_archivebox_server(tmp_path, env=env, port=port)
session = requests.Session()
get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
login_page = session.get(
f"http://admin.archivebox.localhost:{port}/admin/login/",
timeout=10,
)
assert login_page.status_code == 200
csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
assert csrf_match, login_page.text[:500]
login_response = session.post(
f"http://admin.archivebox.localhost:{port}/admin/login/",
headers={"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/"},
data={
"username": "apitestadmin",
"password": "testpass123",
"csrfmiddlewaretoken": csrf_match.group(1),
"next": "/add/",
},
timeout=10,
allow_redirects=False,
)
assert login_response.status_code in (302, 303), login_response.text
_add_page, add_csrf_token = _get_add_view(session, port, host=f"admin.archivebox.localhost:{port}")
session, _add_page, add_csrf_token = _login_to_add_view(port)
response = session.post(
f"http://admin.archivebox.localhost:{port}/add/",
headers={"Referer": f"http://admin.archivebox.localhost:{port}/add/"},
f"http://127.0.0.1:{port}/add/",
headers={
"Host": f"admin.archivebox.localhost:{port}",
"Referer": f"http://admin.archivebox.localhost:{port}/add/",
},
data={
"url": recursive_test_site["root_url"],
"depth": "0",
@@ -686,37 +678,18 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv
try:
start_archivebox_server(tmp_path, env=env, port=port)
session = requests.Session()
get_http_response(port, host=f"admin.archivebox.localhost:{port}", path="/admin/login/")
login_page = session.get(
f"http://admin.archivebox.localhost:{port}/admin/login/",
timeout=10,
)
assert login_page.status_code == 200
csrf_match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', login_page.text)
assert csrf_match, login_page.text[:500]
login_response = session.post(
f"http://admin.archivebox.localhost:{port}/admin/login/",
headers={"Referer": f"http://admin.archivebox.localhost:{port}/admin/login/"},
data={
"username": "apitestadmin",
"password": "testpass123",
"csrfmiddlewaretoken": csrf_match.group(1),
"next": "/add/",
},
timeout=10,
allow_redirects=False,
)
assert login_response.status_code in (302, 303), login_response.text
add_page, add_csrf_token = _get_add_view(session, port, host=f"admin.archivebox.localhost:{port}")
session, add_page, add_csrf_token = _login_to_add_view(port)
assert add_page.status_code == 200
assert 'name="depth"' in add_page.text
assert 'name="url"' in add_page.text
_stop_worker(tmp_path, "worker_runner")
response = session.post(
f"http://admin.archivebox.localhost:{port}/add/",
headers={"Referer": f"http://admin.archivebox.localhost:{port}/add/"},
f"http://127.0.0.1:{port}/add/",
headers={
"Host": f"admin.archivebox.localhost:{port}",
"Referer": f"http://admin.archivebox.localhost:{port}/add/",
},
data={
"url": recursive_test_site["root_url"],
"depth": "2",
@@ -781,14 +754,16 @@ def test_add_view_depth_two_crawl_renders_outputs_over_server(tmp_path, recursiv
assert "active_crawls" in progress.json()
index_page = requests.get(
f"http://web.archivebox.localhost:{port}/public/",
f"http://127.0.0.1:{port}/public/",
headers={"Host": f"web.archivebox.localhost:{port}"},
timeout=10,
)
assert index_page.status_code == 200
assert recursive_test_site["root_url"] in index_page.text
snapshot_admin = session.get(
f"http://admin.archivebox.localhost:{port}/admin/core/snapshot/",
f"http://127.0.0.1:{port}/admin/core/snapshot/",
headers={"Host": f"admin.archivebox.localhost:{port}"},
timeout=10,
)
assert snapshot_admin.status_code == 200
+1 -1
View File
@@ -154,7 +154,7 @@ def running_process_record(initialized_archive):
try:
os_process = psutil.Process(popen.pid)
process = Process.objects.create(
machine=Machine.current(refresh=True),
machine=Machine.current(),
process_type=Process.TypeChoices.ORCHESTRATOR,
pwd=str(initialized_archive),
cmd=cmd,
+1 -1
View File
@@ -35,7 +35,7 @@ def admin_request(admin_user):
@pytest.fixture
def machine():
return Machine.current(refresh=True)
return Machine.current()
@pytest.mark.django_db(transaction=True)
+285 -2
View File
@@ -1,6 +1,10 @@
"""Live progress UI tests."""
import json
import os
import re
import subprocess
import time
import uuid
from datetime import datetime, timezone as dt_timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -15,12 +19,48 @@ from django.urls import reverse
from django.utils import timezone
from archivebox.tests.conftest import ADMIN_TEST_HOST
from archivebox.tests.conftest import cli_env, resolve_abxpkg_binary_env, run_archivebox_cmd
from archivebox.tests.conftest import REPO_ROOT, cli_env, resolve_abxpkg_binary_env, resolve_abxpkg_chrome_env, run_archivebox_cmd
from archivebox.tests.test_archive_result_service import _run_shipped_snapshot_hook
pytestmark = pytest.mark.django_db(transaction=True)
@pytest.fixture
def live_screenshot_command(snapshot, live_server, cached_abxpkg_lib_dir, tmp_path):
from urllib.parse import urlsplit
from archivebox.machine.models import Machine
port = urlsplit(live_server.url).port
machine = Machine.current()
machine.config = {**machine.config, "BASE_URL": f"http://archivebox.localhost:{port}"}
machine.save(update_fields=["config"])
snapshot.url = "https://sweeting.me"
snapshot.permissions = "public"
snapshot.save(update_fields=["url", "permissions"])
Path(snapshot.output_dir).mkdir(parents=True, exist_ok=True)
binary_env = resolve_abxpkg_chrome_env(cached_abxpkg_lib_dir)
variants = [
{"path": str(tmp_path / f"{name}.png"), "width": width, "height": height}
for name, width, height in [("desktop", 1600, 1000), ("tablet", 1024, 1366), ("mobile", 390, 844)]
]
env = {
**os.environ,
**binary_env,
"NODE_PATH": str(cached_abxpkg_lib_dir / "pnpm/packages/chrome/node_modules"),
"SCREENSHOT_HOST_RESOLVER_RULES": "MAP *.archivebox.localhost 127.0.0.1",
"SCREENSHOT_REQUIRE_LIVE_PROGRESS": "1",
"SCREENSHOT_VARIANTS_JSON": json.dumps(variants),
}
command = [
binary_env["NODE_BINARY"],
str(REPO_ROOT / "bin/take_screenshot.js"),
f"http://web.archivebox.localhost:{port}{snapshot.get_absolute_url()}",
variants[0]["path"],
]
return command, env, variants
@pytest.fixture
def painted_http_server():
"""Serve a complete static page so CDP stops emitting changed frames."""
@@ -141,6 +181,249 @@ def real_crawl_setup_process(snapshot, hermetic_lib_dir):
class TestLiveProgressView:
def test_live_progress_uses_existing_indexes_and_cached_totals(self, client, admin_user, crawl, snapshot):
from django.db import connection
from django.test.utils import CaptureQueriesContext
from archivebox.core.models import ArchiveResult, Snapshot
from archivebox.crawls.models import Crawl
Crawl.objects.filter(pk=crawl.pk).update(status=Crawl.StatusChoices.STARTED)
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.SEALED, downloaded_at=None, output_size=1234)
ArchiveResult.objects.create(snapshot=snapshot, plugin="search_backend_sonic", status=ArchiveResult.StatusChoices.QUEUED)
client.force_login(admin_user)
with CaptureQueriesContext(connection) as queries:
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
query_sql = [query["sql"] for query in queries]
assert response.status_code == 200, response.content
payload = response.json()
active_crawl = next(item for item in payload["active_crawls"] if item["id"] == str(crawl.pk))
assert active_crawl["total_snapshots"] == 1
assert active_crawl["crawl_output_size"] is None
assert active_crawl["crawl_output_size_display"] == "…"
deadline = time.monotonic() + 5
while active_crawl["crawl_output_size"] is None and time.monotonic() < deadline:
time.sleep(0.05)
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200, response.content
active_crawl = next(item for item in response.json()["active_crawls"] if item["id"] == str(crawl.pk))
assert active_crawl["cancelled_snapshots"] == 1
assert active_crawl["crawl_output_size"] == 1234
assert payload["indexing_queued"] == 1
assert not any("SUM(" in sql or 'COUNT("core_snapshot"."downloaded_at")' in sql for sql in query_sql)
if connection.vendor == "sqlite":
with connection.cursor() as cursor:
cursor.execute("SELECT name FROM sqlite_master WHERE type='index'")
indexes = {row[0] for row in cursor.fetchall()}
assert "archiveresult_progress_idx" not in indexes
assert "snapshot_progress_totals_idx" not in indexes
aggregates = [sql for sql in query_sql if "GROUP BY" in sql]
assert aggregates
with connection.cursor() as cursor:
for sql in aggregates:
cursor.execute("EXPLAIN QUERY PLAN " + sql)
plan = " ".join(str(row) for row in cursor.fetchall())
if 'FROM "core_snapshot"' in sql:
assert "COVERING INDEX" in plan, plan
else:
# Both existing status indexes can satisfy this lookup.
# Require a status-keyed search, not one planner choice's
# index name; a full scan must still fail below.
assert re.search(r"SEARCH core_archiveresult USING (?:COVERING )?INDEX \S+ \(status=\?", plan), plan
assert "SCAN " not in plan, plan
# Size/cancellation samples are shared by tabs while status counts stay
# current on every request.
Snapshot.objects.filter(pk=snapshot.pk).update(output_size=4321)
with CaptureQueriesContext(connection) as cached_queries:
for _ in range(5):
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200, response.content
cached_crawl = next(item for item in response.json()["active_crawls"] if item["id"] == str(crawl.pk))
assert cached_crawl["crawl_output_size"] == 1234
assert not any("SUM(" in query["sql"] for query in cached_queries)
def test_live_progress_samples_batches_without_leaking_other_snapshots(self, client, admin_user, crawl, snapshot):
from datetime import timedelta
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
Crawl.objects.filter(pk=crawl.pk).update(status=Crawl.StatusChoices.STARTED)
now = timezone.now()
Snapshot.objects.bulk_create(
[
Snapshot(
crawl=crawl,
url=f"https://example.com/batch/{i}",
timestamp=str(now.timestamp() + i + 1),
status=Snapshot.StatusChoices.SEALED,
downloaded_at=now if i % 2 else None,
output_size=i + 1,
)
for i in range(305)
],
)
# Exercise both a timestamp tie crossing batch boundaries and advancing
# to a new timestamp. A started snapshot must not enter sealed totals.
Snapshot.objects.filter(crawl=crawl, status=Snapshot.StatusChoices.SEALED).update(modified_at=now)
Snapshot.objects.filter(crawl=crawl, output_size__gt=250).update(modified_at=now + timedelta(seconds=1))
Snapshot.objects.filter(pk=snapshot.pk).update(output_size=9999)
client.force_login(admin_user)
deadline = time.monotonic() + 5
while True:
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200, response.content
totals = next(item for item in response.json()["active_crawls"] if item["id"] == str(crawl.pk))
if totals["crawl_output_size"] is not None or time.monotonic() >= deadline:
break
time.sleep(0.05)
assert totals["crawl_output_size"] == sum(range(1, 306))
assert totals["cancelled_snapshots"] == 153
assert totals["total_snapshots"] == 306
assert totals["completed_snapshots"] == 305
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.SEALED, config={"PERMISSIONS": "public"})
client.logout()
deadline = time.monotonic() + 5
while True:
response = client.get(reverse("live_progress"), {"snapshot_id": str(snapshot.pk)}, HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200, response.content
scoped = response.json()["active_crawls"][0]
if scoped["crawl_output_size"] is not None or time.monotonic() >= deadline:
break
time.sleep(0.05)
assert scoped["total_snapshots"] == 1
assert scoped["crawl_output_size"] == 9999
assert scoped["cancelled_snapshots"] == 1
def test_live_progress_does_not_inspect_archive_files(self, client, admin_user, crawl, snapshot):
import sys
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.config.constants import CONSTANTS
Crawl.objects.filter(pk=crawl.pk).update(status=Crawl.StatusChoices.STARTED)
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.STARTED)
client.force_login(admin_user)
# Warm imports/settings, then observe real calls without replacing any
# filesystem API or the endpoint under test.
assert client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST).status_code == 200
filesystem_calls = []
storage_roots = (CONSTANTS.ARCHIVE_DIR, CONSTANTS.PERSONAS_DIR)
def observe(frame, event, arg):
if event == "call" and frame.f_code.co_name in {"stat", "lstat", "open", "iterdir", "glob", "rglob"}:
path = frame.f_locals.get("self")
if isinstance(path, Path) and any(path.is_relative_to(root) for root in storage_roots):
filesystem_calls.append(str(path))
if event == "c_call" and arg in (os.scandir, os.listdir):
filesystem_calls.append(arg.__name__)
previous_profile = sys.getprofile()
sys.setprofile(observe)
try:
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
finally:
sys.setprofile(previous_profile)
assert response.status_code == 200, response.content
assert filesystem_calls == []
def test_live_progress_is_read_only_with_expired_machine(self, client, admin_user, crawl, snapshot):
from datetime import timedelta
from django.db import connection
from django.test.utils import CaptureQueriesContext
import archivebox.machine.models as machine_models
from archivebox.machine.models import Machine
machine = Machine.current()
old_modified_at = timezone.now() - timedelta(days=8)
Machine.objects.filter(pk=machine.pk).update(modified_at=old_modified_at)
machine_models._CURRENT_MACHINE = None
client.force_login(admin_user)
with CaptureQueriesContext(connection) as queries:
response = client.get(reverse("live_progress"), HTTP_HOST=ADMIN_TEST_HOST)
assert response.status_code == 200, response.content
writes = [query["sql"] for query in queries if query["sql"].lstrip().upper().startswith(("INSERT", "UPDATE", "DELETE", "REPLACE"))]
assert writes == []
machine.refresh_from_db()
assert machine.modified_at == old_modified_at
def test_gallery_capture_fails_without_live_screencast(self, live_screenshot_command):
command, env, variants = live_screenshot_command
started = time.monotonic()
result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=75)
elapsed = time.monotonic() - started
assert result.returncode != 0, result.stdout
assert "Live progress with a decoded screencast frame was not visible within 60 seconds" in result.stderr
assert 60 <= elapsed < 75, (elapsed, result.stderr)
assert not any(Path(variant["path"]).exists() for variant in variants)
def test_gallery_captures_real_sweeting_screencast(self, crawl, snapshot, live_screenshot_command):
from struct import unpack
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
from archivebox.services.runner import run_due_snapshot
config = {
"PLUGINS": "chrome,chrome_screencast,title,wget,screenshot,dom,pdf",
"PERMISSIONS": "public",
"CHROME_HEADLESS": True,
"CHROME_SANDBOX": False,
"CHROME_ISOLATION": "snapshot",
"CHROME_TIMEOUT": 90,
"CHROME_PAGELOAD_TIMEOUT": 90,
"CHROME_DELAY_AFTER_LOAD": 30,
"TIMEOUT": 90,
}
Crawl.objects.filter(pk=crawl.pk).update(
status=Crawl.StatusChoices.STARTED,
retry_at=timezone.now(),
urls=snapshot.url,
config=config,
)
Snapshot.objects.filter(pk=snapshot.pk).update(status=Snapshot.StatusChoices.QUEUED, retry_at=timezone.now(), config=config)
snapshot = Snapshot.objects.get(pk=snapshot.pk)
errors = []
def run_snapshot():
try:
assert run_due_snapshot(snapshot, lock_seconds=120) is True
except BaseException as err:
errors.append(err)
runner = Thread(target=run_snapshot, name="archivebox-test-marketing-capture")
command, env, variants = live_screenshot_command
capture = subprocess.Popen(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
runner.start()
try:
stdout, stderr = capture.communicate(timeout=75)
assert capture.returncode == 0, stderr
report = json.loads(stdout)
assert report["checks"]["screencastImageLoaded"] is True
captures = report["checks"]["liveProgressCaptures"]
assert len(captures) == len(variants)
for saved, variant in zip(captures, variants, strict=True):
assert saved["ready"] is True, saved
assert saved["screencastImageHasContent"] is True, saved
assert saved["snapshotId"] == str(snapshot.pk).replace("-", "")
assert saved["runningExtractors"] > 0
assert saved["screencastImageUrl"].split("?")[0].endswith(f"/crawl/{crawl.pk}/files/chrome_screencast/latest.jpg")
png = Path(variant["path"]).read_bytes()
assert png[:8] == b"\x89PNG\r\n\x1a\n"
assert unpack(">II", png[16:24]) == (variant["width"], variant["height"])
Path(variants[0]["path"]).with_suffix(".json").write_text(stdout)
finally:
if capture.poll() is None:
capture.terminate()
capture.communicate(timeout=10)
runner.join(timeout=120)
assert not runner.is_alive()
assert errors == []
def test_collection_summary_reports_stored_sizes(self, client, admin_user, snapshot):
import time
from django.core.cache import cache
@@ -558,7 +841,7 @@ class TestLiveProgressView:
try:
os_process = psutil.Process(popen.pid)
Process.objects.create(
machine=Machine.current(refresh=True),
machine=Machine.current(),
process_type=Process.TypeChoices.ORCHESTRATOR,
status=Process.StatusChoices.RUNNING,
pid=popen.pid,
+102
View File
@@ -421,6 +421,12 @@ class TestPublicIndex:
response = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
result_queries = [query["sql"].lower() for query in captured_queries if "core_archiveresult" in query["sql"].lower()]
row_lookups = [
query["sql"]
for query in captured_queries
if re.search(r'WHERE "(?:core_snapshot|crawls_crawl|auth_user)"\."id" =', query["sql"])
]
assert row_lookups == [], "Public rows must reuse the page's snapshot, crawl, and owner metadata"
assert len(result_queries) == 1
assert "output_files" in result_queries[0]
assert "output_str" in result_queries[0]
@@ -435,6 +441,102 @@ class TestPublicIndex:
assert b'data-tooltip="singlefile"' in response.content
assert b'title="singlefile"' not in response.content
@override_settings(PUBLIC_INDEX=True)
@pytest.mark.django_db(transaction=True)
def test_public_count_uses_permission_index_for_uniform_visibility(self, client, admin_user):
import time
from django.core.cache import cache
from django.db import connection
from django.db.backends.signals import connection_created
from django.test.utils import CaptureQueriesContext
from archivebox.core.models import Snapshot
from archivebox.crawls.models import Crawl
crawl = Crawl.objects.create(urls="https://public-count.example", created_by=admin_user, config={"PERMISSIONS": "public"})
Snapshot.objects.bulk_create(
[
Snapshot(
url=f"https://public-count.example/{index}",
timestamp=str(1800000000 + index),
crawl=crawl,
config={"PERMISSIONS": "public"},
status="sealed",
)
for index in range(125)
],
)
if connection.vendor == "sqlite":
with connection.cursor() as cursor:
cursor.execute("ANALYZE core_snapshot")
key = "progress-collection:public"
cache.delete_many([key, f"{key}:refresh"])
background_queries = []
def observe(execute, sql, params, many, context):
background_queries.append((sql, params))
return execute(sql, params, many, context)
def observe_background_connection(sender, connection, **kwargs):
connection.execute_wrappers.append(observe)
connection_created.connect(observe_background_connection)
try:
with CaptureQueriesContext(connection) as queries:
response = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
assert response.status_code == 200
assert not any("COUNT(" in q["sql"] for q in queries), "Public page loads must not wait for a count"
assert response.context["public_snapshot_count"] is None
assert len(response.context["object_list"]) == 50
assert response.context["page_obj"].has_next()
assert b"of 125" not in response.content
deadline = time.monotonic() + 5
while cache.get(key) is None and time.monotonic() < deadline:
time.sleep(0.01)
assert cache.get(key)["snapshots"] == 125
reloaded = client.get("/public/", HTTP_HOST=WEB_TEST_HOST)
assert reloaded.context["public_snapshot_count"] == 125
assert b"of 125" in reloaded.content
last_page = client.get("/public/?page=3", HTTP_HOST=WEB_TEST_HOST)
assert len(last_page.context["object_list"]) == 25
assert not last_page.context["page_obj"].has_next()
assert client.get("/public/?page=4", HTTP_HOST=WEB_TEST_HOST).status_code == 404
# The runner can add a page before the cached total refreshes.
# Navigation follows actual rows, never the old cached page count.
Snapshot.objects.bulk_create(
[
Snapshot(
url=f"https://public-count.example/{index}",
timestamp=str(1800000000 + index),
crawl=crawl,
config={"PERMISSIONS": "public"},
status="sealed",
)
for index in range(125, 160)
],
)
added_page = client.get("/public/?page=4", HTTP_HOST=WEB_TEST_HOST)
assert added_page.status_code == 200
assert len(added_page.context["object_list"]) == 10
assert not added_page.context["page_obj"].has_next()
assert added_page.context["public_snapshot_count"] == 160
assert cache.get(key)["snapshots"] == 125
count_queries = [(sql, params) for sql, params in background_queries if "COUNT(" in sql and '"core_snapshot"' in sql]
assert len(count_queries) == 1, "Reloads must reuse the cached total"
if connection.vendor == "sqlite":
with connection.cursor() as cursor:
cursor.execute("EXPLAIN QUERY PLAN " + count_queries[0][0], count_queries[0][1])
plan = " ".join(str(row[-1]) for row in cursor.fetchall())
assert "SEARCH core_snapshot USING INDEX" in plan and "permissions>" in plan, plan
finally:
connection_created.disconnect(observe_background_connection)
@override_settings(PUBLIC_INDEX=True)
def test_upgraded_07_snapshot_icon_target_renders_at_extensionless_live_url(self, client, admin_user):
from archivebox.core.models import ArchiveResult, Snapshot
+66 -57
View File
@@ -11,6 +11,7 @@ import subprocess
import sys
import threading
import time
from contextlib import ExitStack
from functools import cache
from pathlib import Path
from typing import cast
@@ -1460,6 +1461,7 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep
next_ownership_check = 0.0
try:
while follow:
raise_if_shutdown_requested()
now = time.monotonic()
if keep_running is not None and now >= next_ownership_check:
if not keep_running():
@@ -1476,6 +1478,9 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep
# Read ALL available lines from all files (not just one per iteration)
for log_path, f in file_handles:
while True:
# A noisy worker may never reach EOF; shutdown must not
# depend on returning to the outer poll loop.
raise_if_shutdown_requested()
line = f.readline()
if not line:
break # No more lines available in this file
@@ -1489,10 +1494,8 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None, keep
if not had_output:
time.sleep(0.05)
except (KeyboardInterrupt, BrokenPipeError, OSError):
except (KeyboardInterrupt, SystemExit, OSError):
return "interrupted" # Let the caller handle the cleanup message
except SystemExit:
return "interrupted"
finally:
# Close all file handles
for _, f in file_handles:
@@ -1678,38 +1681,46 @@ def start_server_workers(
config = get_config()
shutdown_state = None
tail_result = "stopped"
try:
supervisor = get_or_create_supervisord_process(daemonize=daemonize)
workers, log_files, components = build_server_worker_plan(
config=config,
host=host,
port=port,
debug=debug,
reload=reload,
nothreading=nothreading,
supervisor=supervisor,
)
component_list = format_runtime_components(components)
if resumed_from_pid:
print(
"[yellow][*] Other newer archivebox process "
f"(pid={resumed_from_pid}) exited, taking over {component_list} in this process again...[/yellow]",
)
else:
print(f"[*] Starting {component_list} in this process (pid={os.getpid()})...")
print()
sync_supervisord_workers(supervisor, workers, prune=True)
print()
if daemonize:
return None
from django.db import connections
connections.close_all()
# Enter cooperative handling only once startup finishes, but retain it
# through our cleanup finally: Popen.wait() has the same interruption risk
# as poll(). ExitStack preserves the existing startup and daemon policies.
with ExitStack() as shutdown_context:
try:
with foreground_shutdown_signals() as shutdown_state:
supervisor = get_or_create_supervisord_process(daemonize=daemonize)
workers, log_files, components = build_server_worker_plan(
config=config,
host=host,
port=port,
debug=debug,
reload=reload,
nothreading=nothreading,
supervisor=supervisor,
)
component_list = format_runtime_components(components)
if resumed_from_pid:
print(
"[yellow][*] Other newer archivebox process "
f"(pid={resumed_from_pid}) exited, taking over {component_list} in this process again...[/yellow]",
)
else:
print(f"[*] Starting {component_list} in this process (pid={os.getpid()})...")
print()
sync_supervisord_workers(supervisor, workers, prune=True)
print()
if daemonize:
return None
from django.db import connections
connections.close_all()
# Immediate exceptions can strand Popen's wait lock inside poll().
# Use the shared cooperative contract in core.shutdown_util; the
# tail checks intent between operations, including while draining
# busy logs. No interruption checks run inside child cleanup.
shutdown_state = shutdown_context.enter_context(foreground_shutdown_signals(raise_on_first_signal=False))
try:
# Tail worker logs while supervisord runs.
sys.stdout.write("Tailing worker logs (Ctrl+C to stop)...\n\n")
sys.stdout.flush()
@@ -1719,26 +1730,24 @@ def start_server_workers(
proc=_supervisord_proc, # Stop tailing when supervisord exits
keep_running=keep_running,
)
except (KeyboardInterrupt, BrokenPipeError, OSError):
if daemonize:
raise
if not shutdown_state or not shutdown_state.signal_name:
print("\n[🛑] Got CTRL+C, stopping gracefully...")
except SystemExit:
if daemonize:
raise
except BaseException as e:
if daemonize:
raise
STDERR.print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping gracefully...")
finally:
signal_shutdown_requested = bool(shutdown_state and shutdown_state.signal_name)
if not daemonize and (signal_shutdown_requested or should_stop_supervisord is None or should_stop_supervisord()):
# Ensure supervisord and all children are stopped only while this
# foreground parent is still the active server parent. Standby
# parents must not tear down a newer leader's services. If this
# foreground parent itself received an OS shutdown signal, always
# stop the supervisord child it owns; stop_own_supervisord_process()
# does not target supervisord processes owned by other parents.
stop_own_supervisord_process(record_exit=not signal_shutdown_requested)
return tail_result
except (KeyboardInterrupt, OSError):
if not shutdown_state.signal_name:
print("\n[🛑] Got CTRL+C, stopping gracefully...")
except SystemExit:
pass
except BaseException as e:
STDERR.print(f"\n[🛑] Got {e.__class__.__name__} exception, stopping gracefully...")
finally:
if not daemonize:
signal_shutdown_requested = bool(shutdown_state and shutdown_state.signal_name)
stop_owned = signal_shutdown_requested or should_stop_supervisord is None or should_stop_supervisord()
# The ownership callback can itself receive the first signal.
signal_shutdown_requested = bool(shutdown_state and shutdown_state.signal_name)
if stop_owned or signal_shutdown_requested:
# A standby parent must never tear down the replacement
# leader's services. Signals stop only our own Popen child,
# even if leadership changed; they avoid DB exit writes.
stop_own_supervisord_process(record_exit=not signal_shutdown_requested)
# A signal can arrive during the final ownership/process-exit check. Do not
# send the outer server loop back into standby or restart after shutdown.
return "interrupted" if shutdown_state and shutdown_state.signal_name else tail_result
+61 -188
View File
@@ -118,10 +118,22 @@ trap cleanup EXIT INT TERM
mkdir -p "$DATA_DIR" "$OUTPUT_DIR" "$PUBLIC_OUTPUT_DIR"
PLUGIN_SCREENSHOT_PLAN="$CAPTURE_ROOT/plugin-screenshots.json"
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" plugin-plan >"$PLUGIN_SCREENSHOT_PLAN"
PLUGIN_ENABLED_CONFIG=()
while IFS= read -r enabled_config; do
[[ -z "$enabled_config" ]] || PLUGIN_ENABLED_CONFIG+=("$enabled_config")
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
'import json,sys; print("\n".join(f"{plugin.upper()}_ENABLED=True" for plugin in json.load(open(sys.argv[1]))["enabled"]))' \
"$PLUGIN_SCREENSHOT_PLAN")
echo "[*] Initializing the ArchiveBox collection at $DATA_DIR"
(
cd "$DATA_DIR"
uv run --no-cache --project "$REPO_DIR" archivebox init --quick
if [[ "${#PLUGIN_ENABLED_CONFIG[@]}" != "0" ]]; then
uv run --no-cache --project "$REPO_DIR" archivebox config --set "${PLUGIN_ENABLED_CONFIG[@]}"
fi
uv run --no-cache --project "$REPO_DIR" archivebox install
)
@@ -145,7 +157,7 @@ if [[ "$HAS_USEFUL_SNAPSHOT" == "0" ]]; then
--overwrite \
--tag=documentation,reference \
--plugins=title,headers,wget,screenshot,pdf,dom,readability,htmltotext,hashes,dns \
https://example.com https://archivebox.io
https://docs.sweeting.me/s/cookie-dilemma https://archivebox.io
)
fi
@@ -209,11 +221,7 @@ PYPORT
)"
(
cd "$DATA_DIR"
uv run --no-cache --project "$REPO_DIR" archivebox config --set \
OPENCODE_ENABLED=True "OPENCODE_PORT=$OPENCODE_PORT" TLSNOTARY_ENABLED=True \
HASHES_ENABLED=True OPENTIMESTAMPS_ENABLED=True
uv run --no-cache --project "$REPO_DIR" archivebox install opencode --binproviders=env,pnpm
uv run --no-cache --project "$REPO_DIR" archivebox install opentimestamps
uv run --no-cache --project "$REPO_DIR" archivebox config --set "OPENCODE_PORT=$OPENCODE_PORT"
)
echo "[*] Starting ArchiveBox on port $PORT"
@@ -262,43 +270,16 @@ VIEWS=(
"Login|$ADMIN_BASE_URL/admin/login/|/admin/login/|archivebox/templates/admin/login.html"
"Public snapshot list|$PUBLIC_BASE_URL/public/|/public/|archivebox/core/views.py"
)
REQUIRED_TEMPLATE_PLUGINS=(
accessibility
chrome
consolelog
defuddle
dns
hashes
headers
htmltotext
liteparse
opendataloader
parse_dom_outlinks
parse_html_urls
parse_jsonl_urls
parse_netscape_urls
parse_rss_urls
parse_txt_urls
redirects
seo
sslcerts
title
trafilatura
)
DISCOVERED_TEMPLATE_PLUGINS=()
add_supplementary_template_capture() {
local plugin_name="$1"
local plugin_names="$1"
local source_url="$2"
local capture_plugins="$plugin_name"
local capture_log="$CAPTURE_ROOT/${plugin_name}-supplementary-capture.log"
local supplementary_plugins=("$plugin_name")
if [[ "$plugin_name" == "opendataloader" ]]; then
capture_plugins="wget"
supplementary_plugins=(liteparse opendataloader)
fi
echo "[*] Capturing real $plugin_name source for template coverage"
local prepare_plugins="$3"
local capture_plugins="${prepare_plugins:-$plugin_names}"
local capture_log="$CAPTURE_ROOT/${plugin_names}-supplementary-capture.log"
local supplementary_plugins=()
IFS=',' read -r -a supplementary_plugins <<<"$plugin_names"
echo "[*] Capturing real $plugin_names source for template coverage"
if ! (
cd "$DATA_DIR"
uv run --no-cache --project "$REPO_DIR" archivebox add \
@@ -308,7 +289,7 @@ add_supplementary_template_capture() {
--plugins="$capture_plugins" \
"$source_url"
) >"$capture_log" 2>&1; then
echo "[!] Supplementary $plugin_name capture failed" >&2
echo "[!] Supplementary $plugin_names capture failed" >&2
tail -100 "$capture_log" >&2
exit 1
fi
@@ -321,37 +302,37 @@ add_supplementary_template_capture() {
'import os; from urllib.parse import urlsplit; from archivebox.core.models import Snapshot; from archivebox.core.routes_util import build_snapshot_detail_url; snapshot=Snapshot.objects.filter(url=os.environ["UI_SCREENSHOT_SOURCE_URL"]).order_by("-bookmarked_at").first(); url=build_snapshot_detail_url(snapshot.archive_path_from_db) if snapshot else ""; print(f"{snapshot.id}\t{url}\t{urlsplit(url).path}" if snapshot else "")'
) | tail -1)"
if [[ -z "$snapshot_record" ]]; then
echo "[!] Supplementary $plugin_name capture did not create a snapshot" >&2
echo "[!] Supplementary $plugin_names capture did not create a snapshot" >&2
tail -100 "$capture_log" >&2
exit 1
fi
local snapshot_id snapshot_view_url snapshot_view_path
IFS=$'\t' read -r snapshot_id snapshot_view_url snapshot_view_path <<<"$snapshot_record"
if [[ "$plugin_name" == "opendataloader" ]]; then
if [[ -n "$prepare_plugins" ]]; then
if ! (
cd "$DATA_DIR"
LITEPARSE_ENABLED=True OPENDATALOADER_ENABLED=True uv run --no-cache --project "$REPO_DIR" archivebox extract \
--plugins=liteparse,opendataloader \
uv run --no-cache --project "$REPO_DIR" archivebox extract \
--plugins="$plugin_names" \
"$snapshot_id"
) >>"$capture_log" 2>&1; then
echo "[!] LiteParse/OpenDataLoader extraction failed for the captured wget PDF" >&2
echo "[!] $plugin_names extraction failed for the prepared snapshot" >&2
tail -100 "$capture_log" >&2
exit 1
fi
stop_background_runner
fi
local discovery_report="$CAPTURE_ROOT/${plugin_name}-output-discovery.json"
local discovery_report="$CAPTURE_ROOT/${plugin_names}-output-discovery.json"
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
SCREENSHOT_SNAPSHOT_HEADER=collapsed \
SCREENSHOT_WIDTH=1600 \
SCREENSHOT_HEIGHT=1000 \
node "$REPO_DIR/bin/take_screenshot.js" "$snapshot_view_url" "$CAPTURE_ROOT/${plugin_name}-output-discovery.png" >"$discovery_report"
node "$REPO_DIR/bin/take_screenshot.js" "$snapshot_view_url" "$CAPTURE_ROOT/${plugin_names}-output-discovery.png" >"$discovery_report"
local supplementary_plugin discovered_preview_url plugin_already_discovered discovered_plugin
local supplementary_plugin discovered_preview_url capture_mode
for supplementary_plugin in "${supplementary_plugins[@]}"; do
discovered_preview_url="$(
UI_SCREENSHOT_DISCOVERY_REPORT="$discovery_report" UI_SCREENSHOT_EXPECT_PLUGIN="$supplementary_plugin" \
@@ -365,17 +346,10 @@ add_supplementary_template_capture() {
exit 1
fi
plugin_already_discovered=0
for discovered_plugin in "${DISCOVERED_TEMPLATE_PLUGINS[@]}"; do
if [[ "$discovered_plugin" == "$supplementary_plugin" ]]; then
plugin_already_discovered=1
break
fi
done
if [[ "$plugin_already_discovered" == "0" ]]; then
DISCOVERED_TEMPLATE_PLUGINS+=("$supplementary_plugin")
VIEWS+=("Snapshot View ($supplementary_plugin)|$snapshot_view_url#$supplementary_plugin|$snapshot_view_path|archivebox/templates/core/snapshot.html")
fi
capture_mode="$(uv run --no-cache --project "$REPO_DIR" python -c \
'import json,sys; recipe=json.load(open(sys.argv[1]))["overrides"][sys.argv[2]]; text=recipe.get("wait_for_text", ""); print("wait-preview-text:" + text if text else "")' \
"$PLUGIN_SCREENSHOT_PLAN" "$supplementary_plugin")"
VIEWS+=("Snapshot View ($supplementary_plugin)|$snapshot_view_url#$supplementary_plugin|$snapshot_view_path|archivebox/templates/core/snapshot.html|$capture_mode")
done
}
@@ -385,49 +359,19 @@ while [[ "$capture_index" -lt "${#VIEWS[@]}" ]]; do
IFS='|' read -r name url expected_path source capture_mode <<<"$view"
capture_index=$((capture_index + 1))
slug="$(printf '%s' "$name" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]' '-' | sed 's/^-//; s/-$//')"
expected_plugin=""
if [[ "$name" == "Snapshot View ("*")" && "$capture_mode" != "live-progress" && "$capture_mode" != "snapshot-collapsed" ]]; then
expected_plugin="${name#Snapshot View (}"
expected_plugin="${expected_plugin%)}"
fi
echo "[*] $name: $url"
if [[ "$capture_mode" == "snapshot-collapsed" ]]; then
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
SCREENSHOT_SNAPSHOT_HEADER=collapsed \
SCREENSHOT_WIDTH=1600 \
SCREENSHOT_HEIGHT=1000 \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$CAPTURE_ROOT/snapshot-header-state.png" >/dev/null
fi
snapshot_header=expanded
[[ "$name" != "Snapshot View ("* ]] || snapshot_header=collapsed
[[ "$capture_mode" != "snapshot-collapsed" ]] || snapshot_header=collapsed
wait_for_text=""
[[ "$capture_mode" != wait-text:* ]] || wait_for_text="${capture_mode#wait-text:}"
preview_wait_for_text=""
[[ "$capture_mode" != wait-preview-text:* ]] || preview_wait_for_text="${capture_mode#wait-preview-text:}"
view_timing_report=""
while IFS='|' read -r profile viewport_width viewport_height; do
while IFS='|' read -r profile _viewport_width _viewport_height; do
filename="$slug-$profile.png"
capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/$profile"
timing_report_path="$view_timing_report"
capture_env=(
"RESOLUTION=$viewport_width,$viewport_height"
"CHROME_RESOLUTION=$viewport_width,$viewport_height"
"PERSONAS_DIR=$PERSONAS_DIR"
"ACTIVE_PERSONA=$ACTIVE_PERSONA"
"SCREENSHOT_RESOLUTION=$viewport_width,$viewport_height"
"SCREENSHOT_TIMEOUT=120"
"CHROME_WAIT_FOR=load"
"SCREENSHOT_COLLAPSE_FILTERS=1"
)
if [[ "$capture_mode" == wait-replay:* ]]; then
capture_env+=(
"SCREENSHOT_WAIT_FOR_TEXT=${capture_mode#wait-replay:}"
"SCREENSHOT_WAIT_FOR_FRAME_URL=/replay/w/"
)
elif [[ "$capture_mode" == wait-text:* ]]; then
capture_env+=(
"CHROME_WAIT_FOR=domcontentloaded"
"SCREENSHOT_WAIT_FOR_TEXT=${capture_mode#wait-text:}"
)
fi
screenshot_path=""
if [[ "$capture_mode" == "setup-wizard" ]]; then
screenshot_path="$capture_dir/screenshot.png"
@@ -521,12 +465,13 @@ PY
SCREENSHOT_WIDTH=1600 \
SCREENSHOT_HEIGHT=1000 \
SCREENSHOT_VARIANTS_JSON="$live_variants" \
SCREENSHOT_REQUIRE_LIVE_PROGRESS=1 \
SCREENSHOT_SNAPSHOT_HEADER=expanded \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
view_timing_report="$capture_dir/report.json"
timing_report_path="$view_timing_report"
fi
elif [[ -n "$expected_plugin" ]]; then
else
screenshot_path="$capture_dir/screenshot.png"
if [[ "$profile" == "desktop" ]]; then
mkdir -p \
@@ -538,58 +483,16 @@ PY
"$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/desktop/screenshot.png" \
"$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/tablet/screenshot.png" \
"$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile/screenshot.png")"
env "${capture_env[@]}" \
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
SCREENSHOT_WIDTH=1600 \
SCREENSHOT_HEIGHT=1000 \
SCREENSHOT_VARIANTS_JSON="$output_variants" \
SCREENSHOT_COLLAPSE_FILTERS=1 \
SCREENSHOT_SNAPSHOT_HEADER=expanded \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
view_timing_report="$capture_dir/report.json"
timing_report_path="$view_timing_report"
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
"$capture_dir/report.json" "$expected_path"
fi
elif [[ "$profile" == "desktop" || "$capture_mode" == wait-replay:* || -z "${ABXPKG_LIB_DIR:-}" ]]; then
capture_log="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")-$profile-abx-dl.log"
if ! env "${capture_env[@]}" uv run --no-cache --project "$REPO_DIR" abx-dl dl \
--plugins=screenshot \
--timeout=120 \
--dir "$capture_dir" \
"$url" >"$capture_log" 2>&1; then
echo "[!] abx-dl failed while capturing $profile $url" >&2
tail -100 "$capture_log" >&2
exit 1
fi
screenshot_path="$(find "$capture_dir" -type f -path '*/screenshot/screenshot.png' -print -quit)"
screenshot_metadata_path="$(find "$capture_dir" -type f -path '*/screenshot/screenshot.json' -print -quit)"
if [[ -z "$screenshot_metadata_path" || ! -s "$screenshot_metadata_path" ]]; then
echo "[!] Screenshot navigation metadata is missing for $profile $url" >&2
tail -100 "$capture_log" >&2
exit 1
fi
uv run --no-cache --project "$REPO_DIR" "$REPO_DIR/bin/generate_ui_screenshot_gallery.py" validate \
"$screenshot_metadata_path" "$expected_path"
else
screenshot_path="$capture_dir/screenshot.png"
if [[ "$profile" == "tablet" ]]; then
mobile_capture_dir="$CAPTURE_ROOT/$(printf '%02d' "$capture_index")/mobile"
mkdir -p "$capture_dir" "$mobile_capture_dir"
responsive_variants="$(printf \
'[{"path":"%s","width":1024,"height":1366},{"path":"%s","width":390,"height":844}]' \
"$capture_dir/screenshot.png" \
"$mobile_capture_dir/screenshot.png")"
NODE_PATH="$ABXPKG_LIB_DIR/pnpm/packages/chrome/node_modules" \
CHROME_BINARY="$SCREENSHOT_CHROME_BINARY" \
SCREENSHOT_USER_DATA_DIR="$PERSONAS_DIR/$ACTIVE_PERSONA/chrome_profile" \
SCREENSHOT_WIDTH=1024 \
SCREENSHOT_HEIGHT=1366 \
SCREENSHOT_VARIANTS_JSON="$responsive_variants" \
SCREENSHOT_COLLAPSE_FILTERS=1 \
SCREENSHOT_SNAPSHOT_HEADER="$snapshot_header" \
SCREENSHOT_WAIT_FOR_TEXT="$wait_for_text" \
SCREENSHOT_PREVIEW_WAIT_FOR_TEXT="$preview_wait_for_text" \
node "$REPO_DIR/bin/take_screenshot.js" "$url" "$screenshot_path" >"$capture_dir/report.json"
view_timing_report="$capture_dir/report.json"
timing_report_path="$view_timing_report"
@@ -598,8 +501,7 @@ PY
fi
fi
if [[ -z "$screenshot_path" || ! -s "$screenshot_path" ]]; then
echo "[!] Screenshot plugin did not produce a $profile PNG for $url" >&2
[[ -n "${capture_log:-}" && -f "$capture_log" ]] && tail -100 "$capture_log" >&2
echo "[!] Screenshot capture did not produce a $profile PNG for $url" >&2
exit 1
fi
cp "$screenshot_path" "$OUTPUT_DIR/$filename"
@@ -649,19 +551,21 @@ PY
) >"$CAPTURE_ROOT/sweeting-live-capture.log" 2>&1 &
ARCHIVE_PID=$!
# Open the detail page once archiving starts, so its initial state and
# the live monitor both show the capture in progress.
LIVE_SNAPSHOT_VIEW_URL=""
for _attempt in $(seq 1 120); do
LIVE_SNAPSHOT_RECORD="$( (
cd "$DATA_DIR"
UI_SCREENSHOT_CAPTURE_STARTED_AT="$SWEETING_CAPTURE_STARTED_AT" uv run --no-cache --project "$REPO_DIR" archivebox manage shell --no-imports -c \
'import os; from urllib.parse import urlsplit; from django.utils.dateparse import parse_datetime; from archivebox.core.models import Snapshot; from archivebox.core.routes_util import build_snapshot_detail_url; started_at=parse_datetime(os.environ["UI_SCREENSHOT_CAPTURE_STARTED_AT"]); snapshot=Snapshot.objects.filter(url__startswith="https://sweeting.me",bookmarked_at__gte=started_at).order_by("-bookmarked_at").first(); url=build_snapshot_detail_url(snapshot.archive_path_from_db) if snapshot else ""; print(f"{url}\t{urlsplit(url).path}" if snapshot else "")'
'import os; from urllib.parse import urlsplit; from django.utils.dateparse import parse_datetime; from archivebox.core.models import Snapshot; from archivebox.core.routes_util import build_snapshot_detail_url; started_at=parse_datetime(os.environ["UI_SCREENSHOT_CAPTURE_STARTED_AT"]); snapshot=Snapshot.objects.filter(url__startswith="https://sweeting.me",bookmarked_at__gte=started_at,status="started").order_by("-bookmarked_at").first(); url=build_snapshot_detail_url(snapshot.archive_path_from_db) if snapshot else ""; print(f"{url}\t{urlsplit(url).path}" if snapshot else "")'
) | tail -1)"
if [[ -n "$LIVE_SNAPSHOT_RECORD" ]]; then
IFS=$'\t' read -r LIVE_SNAPSHOT_VIEW_URL LIVE_SNAPSHOT_VIEW_PATH <<<"$LIVE_SNAPSHOT_RECORD"
break
fi
if ! kill -0 "$ARCHIVE_PID" 2>/dev/null; then
echo "[!] Sweeting.me capture exited before creating a snapshot" >&2
echo "[!] Sweeting.me capture exited before an active snapshot could be opened" >&2
tail -100 "$CAPTURE_ROOT/sweeting-live-capture.log" >&2
exit 1
fi
@@ -748,54 +652,23 @@ PY
SCREENSHOT_WIDTH=1600 \
SCREENSHOT_HEIGHT=1000 \
node "$REPO_DIR/bin/take_screenshot.js" "$LIVE_SNAPSHOT_VIEW_URL" "$CAPTURE_ROOT/snapshot-output-discovery.png" >"$SNAPSHOT_DISCOVERY_REPORT"
SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --no-cache --project "$REPO_DIR" python -c \
'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); print("\n".join("{}\t{}\t{}".format(output["plugin"], output["outputPath"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"]))')"
SNAPSHOT_OUTPUT_PLUGINS="$(UI_SCREENSHOT_PLAN="$PLUGIN_SCREENSHOT_PLAN" UI_SCREENSHOT_DISCOVERY_REPORT="$SNAPSHOT_DISCOVERY_REPORT" uv run --no-cache --project "$REPO_DIR" python -c \
'import json, os; from urllib.parse import urlsplit; report=json.load(open(os.environ["UI_SCREENSHOT_DISCOVERY_REPORT"])); overrides=json.load(open(os.environ["UI_SCREENSHOT_PLAN"]))["overrides"]; print("\n".join("{}\t{}\t{}".format(output["plugin"], output["outputPath"], "wait-replay:Nick Sweeting" if urlsplit(output["previewUrl"]).path.endswith(".wacz") else "") for output in report["checks"]["snapshotOutputs"] if output["plugin"] not in overrides))')"
if [[ -z "$SNAPSHOT_OUTPUT_PLUGINS" ]]; then
echo "[!] The Sweeting.me snapshot detail page exposed no selectable outputs" >&2
exit 1
fi
for required_plugin in tlsnotary opentimestamps; do
if ! printf '%s\n' "$SNAPSHOT_OUTPUT_PLUGINS" | cut -f1 | grep -qx "$required_plugin"; then
echo "[!] Live capture did not expose the required $required_plugin output" >&2
tail -200 "$CAPTURE_ROOT/sweeting-live-capture.log" >&2
cat "$SNAPSHOT_DISCOVERY_REPORT" >&2
exit 1
fi
done
while IFS=$'\t' read -r plugin_name output_path output_capture_mode; do
[[ -z "$plugin_name" ]] && continue
DISCOVERED_TEMPLATE_PLUGINS+=("$plugin_name")
VIEWS+=("Snapshot View ($plugin_name)|$LIVE_SNAPSHOT_VIEW_URL#$output_path|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|$output_capture_mode")
done <<<"$SNAPSHOT_OUTPUT_PLUGINS"
add_supplementary_template_capture \
parse_jsonl_urls \
https://raw.githubusercontent.com/noho/dayu-agent/f4343e490dd096e81a2f4e715c5592a10e126a26/utils/web_ci_urls.jsonl
add_supplementary_template_capture \
parse_rss_urls \
https://github.com/ArchiveBox/ArchiveBox/releases.atom
add_supplementary_template_capture \
parse_netscape_urls \
https://gist.githubusercontent.com/jgarber623/cdc8e2fa1cbcb6889872/raw/2081aac716fd9aeb2dc443cd7cace57b8e830c57/delicious.html
add_supplementary_template_capture \
opendataloader \
https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf
while IFS='|' read -r example_plugins example_url example_prepare; do
[[ -z "$example_plugins" ]] && continue
add_supplementary_template_capture "$example_plugins" "$example_url" "$example_prepare"
done < <(uv run --no-cache --project "$REPO_DIR" python -c \
'import json,sys; plan=json.load(open(sys.argv[1])); print("\n".join("|".join((",".join(example["plugins"]), example["url"], ",".join(example["prepare_plugins"]))) for example in plan["examples"]))' \
"$PLUGIN_SCREENSHOT_PLAN")
missing_template_plugins=()
for required_plugin in "${REQUIRED_TEMPLATE_PLUGINS[@]}"; do
plugin_was_discovered=0
for discovered_plugin in "${DISCOVERED_TEMPLATE_PLUGINS[@]}"; do
if [[ "$discovered_plugin" == "$required_plugin" ]]; then
plugin_was_discovered=1
break
fi
done
[[ "$plugin_was_discovered" == "0" ]] && missing_template_plugins+=("$required_plugin")
done
if [[ "$MAX_VIEWS" == "0" && "${#missing_template_plugins[@]}" != "0" ]]; then
echo "[!] Real snapshot UI discovery is missing template coverage for: ${missing_template_plugins[*]}" >&2
cat "$SNAPSHOT_DISCOVERY_REPORT" >&2
exit 1
fi
VIEWS+=("Snapshot View (header collapsed)|$LIVE_SNAPSHOT_VIEW_URL|$LIVE_SNAPSHOT_VIEW_PATH|archivebox/templates/core/snapshot.html|snapshot-collapsed")
fi
if [[ "$MAX_VIEWS" != "0" && "$capture_index" -ge "$MAX_VIEWS" ]]; then
+46 -13
View File
@@ -32,7 +32,6 @@ REQUIRED_VIEW_NAMES = {
"Snapshot admin detail",
"Snapshot View (capture in progress)",
"Snapshot View (header collapsed)",
"Snapshot View (opentimestamps)",
"Snapshot files",
"Archive results",
"Archive result detail",
@@ -61,6 +60,41 @@ REQUIRED_VIEW_NAMES = {
}
def plugin_screenshot_plan() -> dict:
"""Discover default coverage and plugin-owned examples from the installed package."""
from abx_plugins import get_plugins_dir
required = []
examples = {}
overrides = {}
enabled_plugins = []
for plugin_dir in sorted(get_plugins_dir().iterdir()):
config_path = plugin_dir / "config.json"
if not config_path.is_file():
continue
config = json.loads(config_path.read_text())
recipe = json.loads((plugin_dir / config["screenshot"]).read_text()) if config.get("screenshot") else {}
plugin = plugin_dir.name
enabled = config.get("properties", {}).get(f"{plugin.upper()}_ENABLED", {}).get("default", False)
if recipe.get("enabled") or recipe.get("url"):
enabled_plugins.append(plugin)
enabled = True
if recipe.get("view"):
required.append(recipe["view"])
elif enabled and (
recipe.get("url")
or (plugin_dir / "templates/full.html").is_file()
or ((plugin_dir / "templates/card.html").is_file() and not config.get("card_hidden"))
):
required.append(f"Snapshot View ({plugin})")
if recipe.get("url"):
overrides[plugin] = recipe
key = (recipe["url"], tuple(recipe.get("prepare_plugins", [])))
example = examples.setdefault(key, {"url": key[0], "prepare_plugins": list(key[1]), "plugins": []})
example["plugins"].append(plugin)
return {"required": required, "examples": list(examples.values()), "overrides": overrides, "enabled": enabled_plugins}
def build_provenance() -> dict[str, str]:
version = tomllib.loads((REPO_DIR / "pyproject.toml").read_text(encoding="utf-8"))["project"]["version"]
revision = os.environ.get("GITHUB_SHA", "").strip()
@@ -213,19 +247,9 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
if not allow_partial:
captured_names = set(captures_by_name)
missing = sorted(REQUIRED_VIEW_NAMES - captured_names)
missing = sorted((REQUIRED_VIEW_NAMES | set(plugin_screenshot_plan()["required"])) - captured_names)
if missing:
raise SystemExit(f"required UI screenshot coverage is missing: {', '.join(missing)}")
snapshot_output_views = [
name
for name in captured_names
if name.startswith("Snapshot View (")
and name not in {"Snapshot View (capture in progress)", "Snapshot View (header collapsed)"}
]
if len(snapshot_output_views) < 20:
raise SystemExit(
f"expected at least 20 rendered Sweeting.me snapshot outputs, got {len(snapshot_output_views)}",
)
markdown_sections = []
html_sections = []
for capture in grouped_captures:
@@ -276,7 +300,7 @@ def build_galleries(manifest_path: Path, markdown_path: Path, html_path: Path) -
),
)
html_sections.append(
f"<article><h2>{html.escape(capture['name'])}</h2>"
f'<article id="{re.sub(r"[^a-z0-9]+", "-", capture["name"].lower()).strip("-")}"><h2>{html.escape(capture["name"])}</h2>'
f'<p><a href="{html.escape(capture["url"])}"><code>{html.escape(route)}</code></a> · '
f'<a href="{html.escape(source_url)}">View code</a>{html.escape(ttfb_text)}</p>'
f'<div class="shots">{"".join(html_figures)}</div></article>',
@@ -345,6 +369,12 @@ def refresh_gallery(html_path: Path) -> None:
if not header or not sections:
raise SystemExit("Published screenshot gallery is missing its capture header or content")
content = sections[1]
content = re.sub(
r"<article><h2>(.*?)</h2>",
lambda match: f'<article id="{re.sub(r"[^a-z0-9]+", "-", html.unescape(match[1]).lower()).strip("-")}"><h2>{match[1]}</h2>',
content,
flags=re.DOTALL,
)
for profile in CAPTURE_PROFILES:
# Galleries published before the switcher used classes alone.
content = content.replace(
@@ -355,6 +385,9 @@ def refresh_gallery(html_path: Path) -> None:
def main() -> None:
if sys.argv[1:] == ["plugin-plan"]:
print(json.dumps(plugin_screenshot_plan()))
return
if len(sys.argv) == 3 and sys.argv[1] == "refresh":
refresh_gallery(Path(sys.argv[2]))
return
+156 -10
View File
@@ -26,6 +26,9 @@ Environment:
SCREENSHOT_FULL_PAGE Set to 1 to capture the full page, defaults to viewport only
SCREENSHOT_SCROLL_SELECTOR Scroll this selector into view before capture
SCREENSHOT_WAIT_SELECTOR Wait for this selector before capture
SCREENSHOT_WAIT_FOR_TEXT Wait for this visible page text before capture
SCREENSHOT_PREVIEW_WAIT_FOR_TEXT Wait for this text in the selected snapshot preview iframe
SCREENSHOT_REQUIRE_LIVE_PROGRESS Require active snapshot progress and a nonblank decoded screencast within 60s
SCREENSHOT_CLICK_SELECTOR Click this selector before capture
SCREENSHOT_AFTER_CLICK_WAIT_SELECTOR Wait for this selector after clicking
SCREENSHOT_HOST_RESOLVER_RULES Chrome host resolver rules
@@ -61,7 +64,7 @@ function addLaunchArg(args, arg) {
if (!args.includes(arg)) args.push(arg);
}
async function waitForVisibleCardPreviews(page) {
async function waitForVisibleCardPreviews(page, timeout = 45000) {
// The page load event does not wait for nested lazy card images. Capture their
// decoded pixels, without forcing hidden alternatives or offscreen cards to load.
const cardHandles = await page.$$('iframe[title$=" card"]');
@@ -106,10 +109,114 @@ async function waitForVisibleCardPreviews(page) {
if (!images.every((img) => img.complete && img.naturalWidth > 0)) return false;
await Promise.all(images.map((img) => img.decode()));
return true;
}, {timeout: 45000}, expectedUrl);
}, {timeout}, expectedUrl);
}));
}
async function liveProgressState(readyOnly = false, rejectedFrameUrl = '') {
const visible = (element, inViewport = true) => {
if (!element || !element.checkVisibility({checkOpacity: true, checkVisibilityCSS: true})) return false;
const rect = element.getBoundingClientRect();
if (rect.width <= 0 || rect.height <= 0) return false;
if (!inViewport) return true;
const left = Math.max(0, rect.left);
const right = Math.min(innerWidth, rect.right);
const top = Math.max(0, rect.top);
const bottom = Math.min(innerHeight, rect.bottom);
return right > left && bottom > top
&& element.contains(document.elementFromPoint((left + right) / 2, (top + bottom) / 2));
};
const monitor = document.querySelector('#progress-monitor[data-progress-scope="snapshot"]');
const panel = monitor?.querySelector('.screencast-panel.visible[data-mode="frame"]');
const img = panel?.querySelector('.screencast-frame img');
const src = img?.currentSrc || '';
const snapshotId = monitor
? new URL(monitor.dataset.progressEndpoint, location.href).searchParams.get('snapshot_id')
: '';
const snapshot = [...(monitor?.querySelectorAll('.snapshot-item') || [])]
.find((item) => item.dataset.snapshotId?.replaceAll('-', '') === snapshotId?.replaceAll('-', ''));
const state = {
url: location.href,
title: document.title,
progressMonitorVisible: visible(monitor),
progressDetailsVisible: visible(monitor?.querySelector('.progress-content')),
snapshotId,
snapshotStarted: Boolean(snapshot?.querySelector('.status-badge.started')),
runningExtractors: [...(snapshot?.querySelectorAll('.extractor-badge.started') || [])]
.filter((element) => visible(element, false)).length,
screencastVisible: visible(panel) && visible(img),
screencastPlaceholderVisible: visible(monitor?.querySelector('.screencast-placeholder')),
screencastImageLoaded: Boolean(img?.complete && img.naturalWidth > 0 && img.naturalHeight > 0),
screencastImageSize: img ? `${img.naturalWidth}x${img.naturalHeight}` : '',
screencastImageUrl: src,
};
state.ready = state.progressMonitorVisible && state.progressDetailsVisible
&& state.snapshotStarted && state.runningExtractors > 0
&& state.screencastVisible && !state.screencastPlaceholderVisible && state.screencastImageLoaded
&& /\/api\/v1\/crawls\/crawl\/[^/]+\/files\/chrome_screencast\/latest\.jpg$/.test(new URL(src || 'about:blank').pathname);
if (!readyOnly) return state;
if (!state.ready || src === rejectedFrameUrl) return false;
try {
await img.decode();
} catch {
return false;
}
return img.isConnected && img.currentSrc === src && visible(img) ? state : false;
}
async function waitForLiveProgress(page, deadline, screencastResponses) {
let rejectedFrameUrl = '';
try {
while (Date.now() < deadline) {
const handle = await page.waitForFunction(liveProgressState, {timeout: Math.max(1, deadline - Date.now()), polling: 100}, true, rejectedFrameUrl);
const state = await handle.jsonValue();
await handle.dispose();
// Chrome's initial frame can be a fully decoded, solid black/white image.
// Inspect the exact response displayed by the UI, without requesting a
// newer latest.jpg or bypassing its normal authentication/CORS behavior.
const response = screencastResponses.get(state.screencastImageUrl);
if (!response) throw new Error('Displayed screencast has no observed image response');
const imageData = `data:image/jpeg;base64,${(await response.buffer()).toString('base64')}`;
const hasContent = await page.evaluate(async (data) => {
const img = new Image();
img.src = data;
await img.decode();
const canvas = document.createElement('canvas');
canvas.width = canvas.height = 64;
const context = canvas.getContext('2d');
// Ignore the outer edges: a site header alone can appear while the
// actual page is still blank. Marketing captures need page content.
context.drawImage(img, img.naturalWidth * 0.1, img.naturalHeight * 0.1,
img.naturalWidth * 0.8, img.naturalHeight * 0.8, 0, 0, 64, 64);
const pixels = context.getImageData(0, 0, 64, 64).data;
// Check each channel separately: a solid colored frame is also blank.
return [0, 1, 2].some((channel) => {
let min = 255;
let max = 0;
for (let i = channel; i < pixels.length; i += 4) {
min = Math.min(min, pixels[i]);
max = Math.max(max, pixels[i]);
}
return max - min > 16;
});
}, imageData);
if (hasContent) {
const current = await page.evaluate(liveProgressState);
if (current.ready && current.screencastImageUrl === state.screencastImageUrl) {
return {...current, screencastImageHasContent: true};
}
} else {
rejectedFrameUrl = state.screencastImageUrl;
}
}
throw new Error('Live progress deadline expired');
} catch (error) {
const state = await page.evaluate(liveProgressState).catch(() => ({url: page.url()}));
if (rejectedFrameUrl) state.rejectedBlankFrameUrl = rejectedFrameUrl;
throw new Error(`Live progress with a decoded screencast frame was not visible within 60 seconds: ${JSON.stringify(state)}`, {cause: error});
}
}
async function main() {
if (process.argv.includes('--help') || process.argv.includes('-h')) {
usage();
@@ -121,6 +228,7 @@ async function main() {
const width = Number(process.env.SCREENSHOT_WIDTH || 1600);
const height = Number(process.env.SCREENSHOT_HEIGHT || 1400);
const fullPage = process.env.SCREENSHOT_FULL_PAGE === '1';
const requireLiveProgress = process.env.SCREENSHOT_REQUIRE_LIVE_PROGRESS === '1';
const variants = process.env.SCREENSHOT_VARIANTS_JSON
? JSON.parse(process.env.SCREENSHOT_VARIANTS_JSON)
: [];
@@ -162,6 +270,14 @@ async function main() {
await page.setCacheEnabled(false);
await Promise.all(restoredPages.map((restoredPage) => restoredPage.close()));
page.setDefaultTimeout(45000);
const screencastResponses = new Map();
if (requireLiveProgress) {
page.on('response', (response) => {
if (/\/files\/chrome_screencast\/latest\.jpg$/.test(new URL(response.url()).pathname) && response.ok()) {
screencastResponses.set(response.url(), response);
}
});
}
const client = await page.createCDPSession();
const documentRequests = new Map();
@@ -209,6 +325,7 @@ async function main() {
await page.setCookie(cookie);
}
const liveProgressDeadline = Date.now() + 60000;
let navigationResponse = await page.goto(url, { waitUntil: 'load', timeout: 60000 });
if (process.env.SCREENSHOT_LOGIN_USERNAME || process.env.SCREENSHOT_LOGIN_PASSWORD) {
@@ -231,10 +348,22 @@ async function main() {
}
await page.waitForSelector('body');
await page.waitForSelector('#progress-monitor, #add-form', { timeout: 5000 }).catch(() => {});
if (process.env.SCREENSHOT_WAIT_FOR_TEXT) {
await page.waitForFunction((text) => document.body.innerText.includes(text), {}, process.env.SCREENSHOT_WAIT_FOR_TEXT);
}
if (process.env.SCREENSHOT_WAIT_SELECTOR) {
await page.waitForSelector(process.env.SCREENSHOT_WAIT_SELECTOR, { timeout: 45000 });
}
if (process.env.SCREENSHOT_PREVIEW_WAIT_FOR_TEXT) {
const previewHandle = await page.waitForSelector('#main-frame', {timeout: 45000});
const preview = await previewHandle.contentFrame();
if (!preview) throw new Error('Snapshot preview iframe has no document');
await preview.waitForFunction(
(text) => document.body?.innerText.includes(text),
{timeout: 45000},
process.env.SCREENSHOT_PREVIEW_WAIT_FOR_TEXT,
);
}
if (process.env.SCREENSHOT_CLICK_SELECTOR) {
await page.waitForSelector(process.env.SCREENSHOT_CLICK_SELECTOR, { timeout: 45000 });
await page.click(process.env.SCREENSHOT_CLICK_SELECTOR);
@@ -249,7 +378,18 @@ async function main() {
}, process.env.SCREENSHOT_SCROLL_SELECTOR);
}
await new Promise((resolve) => setTimeout(resolve, 1200));
if (requireLiveProgress) {
await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
} else {
await new Promise((resolve) => setTimeout(resolve, 1200));
}
// Selecting a hash opens its output stack after the saved header preference
// is read. Apply the requested collapse through the real toggle afterwards.
if (process.env.SCREENSHOT_SNAPSHOT_HEADER === 'collapsed') {
const toggle = await page.$('.header-toggle[aria-expanded="true"]');
if (toggle) await toggle.click();
}
const checks = await page.evaluate(() => ({
url: location.href,
@@ -301,20 +441,26 @@ async function main() {
}
const screenshotPaths = [];
if (requireLiveProgress) checks.liveProgressCaptures = [];
const capture = async (screenshotPath) => {
await waitForVisibleCardPreviews(page, requireLiveProgress ? Math.max(1, liveProgressDeadline - Date.now()) : 45000);
if (requireLiveProgress) {
const liveProgress = await waitForLiveProgress(page, liveProgressDeadline, screencastResponses);
checks.liveProgressCaptures.push({path: screenshotPath, viewport: page.viewport(), ...liveProgress});
}
await page.screenshot({path: screenshotPath, fullPage});
screenshotPaths.push(screenshotPath);
};
if (variants.length) {
for (const variant of variants) {
const variantPath = path.resolve(variant.path);
fs.mkdirSync(path.dirname(variantPath), { recursive: true });
await page.setViewport({width: variant.width, height: variant.height, deviceScaleFactor: 1});
await new Promise((resolve) => setTimeout(resolve, 300));
await waitForVisibleCardPreviews(page);
await page.screenshot({ path: variantPath, fullPage });
screenshotPaths.push(variantPath);
await capture(variantPath);
}
} else {
await waitForVisibleCardPreviews(page);
await page.screenshot({ path: output, fullPage });
screenshotPaths.push(output);
await capture(output);
}
console.log(JSON.stringify({ screenshotPath: screenshotPaths[0], screenshotPaths, checks }, null, 2));
} finally {
-6
View File
@@ -54,7 +54,6 @@ version = 2
"16f996904f0eaa96-1" = "run"
"167cab3db82a6aee-1" = "run"
"6a6234b63cd3d0ae-1" = "run"
"396490664ce459e9-1" = "run"
# archivebox/mcp/README.md
"d9ff84abeb97d380-1" = "run"
@@ -69,7 +68,6 @@ version = 2
"ac2c1555d3b9cd8f-1" = "run"
"11a520069efa3387-1" = "run"
"46fe9d81940ea385-1" = "run"
"396490664ce459e9-2" = "run"
# docs/ArchiveBox-Architecture-Diagrams.md
"7442dd337fd0a1dd-1" = "illustration"
@@ -282,7 +280,6 @@ version = 2
"16f996904f0eaa96-1" = "project"
"167cab3db82a6aee-1" = "system"
"6a6234b63cd3d0ae-1" = "collection"
"396490664ce459e9-1" = "project"
"d9ff84abeb97d380-1" = "collection"
"aebc15190116f5cd-1" = "collection"
"0b307d4f1033723f-1" = "collection"
@@ -290,7 +287,6 @@ version = 2
"ac2c1555d3b9cd8f-1" = "project"
"11a520069efa3387-1" = "system"
"46fe9d81940ea385-1" = "collection"
"396490664ce459e9-2" = "project"
"201988374bff6d14-1" = "collection"
"e305c234db136566-1" = "collection"
"798793d9d6b2e12e-1" = "collection"
@@ -363,7 +359,6 @@ version = 2
"16f996904f0eaa96-1" = "ubuntu"
"167cab3db82a6aee-1" = "ubuntu"
"6a6234b63cd3d0ae-1" = "ubuntu"
"396490664ce459e9-1" = "ubuntu"
"d9ff84abeb97d380-1" = "ubuntu"
"aebc15190116f5cd-1" = "ubuntu"
"0b307d4f1033723f-1" = "ubuntu"
@@ -371,7 +366,6 @@ version = 2
"ac2c1555d3b9cd8f-1" = "ubuntu"
"11a520069efa3387-1" = "ubuntu"
"46fe9d81940ea385-1" = "ubuntu"
"396490664ce459e9-2" = "ubuntu"
"201988374bff6d14-1" = "ubuntu"
"e305c234db136566-1" = "ubuntu"
"798793d9d6b2e12e-1" = "ubuntu"
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "archivebox",
"version": "0.9.71rc3",
"version": "0.9.71rc42",
"repository": "github:ArchiveBox/ArchiveBox",
"license": "MIT",
"dependencies": {
+6 -6
View File
@@ -158,7 +158,7 @@
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
<h3>يحفظ علامات التبويب في ثوان ، يخزنها منذ عقود</h3>
<p>جمع الإشارات المرجعية للقراءة لاحقًا ، أو الاحتفاظ بالأدلة للحالات القانونية التي يتم التحقق منها من عمليات التقاط من موقعنا <a href="https://tlsnotary.zervice.io/">ملحق تسجيل TLSNotary</a>.</p>
<p>جمع الإشارات المرجعية للقراءة لاحقًا ، أو الاحتفاظ بالأدلة للحالات القانونية التي يتم التحقق منها من عمليات التقاط من موقعنا <a href="https://tlsnotary.archivebox.io/">ملحق تسجيل TLSNotary</a>.</p>
</article>
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
@@ -204,7 +204,7 @@
<ul>
<li>تخزين لقطات الشاشة ، ملفات PDFs ، الرؤوس ، ملفات WARC ، واستخراج النص.</li>
<li>ضع علامة على المصادر واستعراضها من خلال واجهة الويب المستضافة ذاتيًا.</li>
<li>استخدام ZK proofs مع <a href="https://tlsnotary.zervice.io/">TLSNotary</a> لإثبات صحة المحتوى.</li>
<li>استخدام ZK proofs مع <a href="https://tlsnotary.archivebox.io/">TLSNotary</a> لإثبات صحة المحتوى.</li>
</ul>
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="مخرج TLSNotary للتحقق من استجابة تم أرشيفة في صفحة تفاصيل اللقطة" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
</article>
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>خادم منزلي</span></div></div><figcaption>على الأجهزة الخاصة بك</figcaption></figure>
<div class="command-card">
<div class="step"><span>1</span><div><h4>اختر منصتك</h4><p>قد تتأخر تكاملات المجتمع عن إصدارات Docker وPython.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">نونريد</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>تثبيت وفتح ArchiveBox</h4><p>اتبع دليل تثبيت المنصة ، قم بتكوين التخزين المستمر ، ثم افتح التطبيق لإنهاء الإعداد.</p><p>بالنسبة إلى TrueNAS، راجع <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">ملاحظات منصة README</a> قبل اختيار التكامل.</p></div></div>
<div class="step"><span>1</span><div><h4>اختر منصتك</h4><p>قد تتأخر تكاملات المجتمع عن إصدارات Docker وPython.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">نونريد</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>تثبيت وفتح ArchiveBox</h4><p>اتبع دليل تثبيت المنصة ، قم بتكوين التخزين المستمر ، ثم افتح التطبيق لإنهاء الإعداد.</p></div></div>
</div>
</div>
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<li><a aria-label="ملحق أحادي الملف" href="https://plugins.archivebox.io/#singlefile" title="ملف فردي"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
<li><a aria-label="يوتيوب/yt-dlp البرنامج المساعد" href="https://plugins.archivebox.io/#ytdlp" title="يوتيوب/yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
<li><a aria-label="ذيل" href="https://tailscale.com/" title="ذيل"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
<li><a aria-label="TLSNotary" href="https://tlsnotary.zervice.io/" title="TLSNotary"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
<li><a aria-label="TLSNotary" href="https://tlsnotary.archivebox.io/" title="TLSNotary"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
</ul>
<p class="configuration-description">يمكن إعداد ArchiveBox عبر واجهة الإدارة أو واجهة سطر الأوامر، <code>ArchiveBox.conf</code>، أو عن طريق استيراد الإعدادات وملفات الكوكيز من Chrome أو متصفحات أخرى. يعمل نموذج التكوين نفسه في Docker وDocker Compose، وفي عمليات التثبيت على الأجهزة الحقيقية، وفي المهام المجدولة، وكذلك في التشغيلات المؤقتة عبر واجهة سطر الأوامر.</p>
</div>
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<!-- ARCHIVEBOX:FOOTER-START -->
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">أدوات مفتوحة المصدر للاحتفاظ بنسخة دائمة من الويب.<br/>بياناتك. أجهزتك. أرشيفك.</p></div><p class="abx-footer-motto">مكان للويب<br/>الذي تريد الاحتفاظ به.</p></div>
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>التطبيقات والأدوات</h2><ul><li><a href="https://archivebox.io/">خادم ArchiveBox</a><a aria-label="خادم ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox لنظامَي macOS وiOS</a><a aria-label="ArchiveBox لنظامَي macOS وiOS — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox لنظام Android</a><a aria-label="ArchiveBox لنظام Android — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox لسطح المكتب</a><a aria-label="ArchiveBox لسطح المكتب — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">إضافة ArchiveBox للمتصفح</a><a aria-label="إضافة ArchiveBox للمتصفح — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">مكتبة إضافات ArchiveBox</a><a aria-label="مكتبة إضافات ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-Line truncated
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>التطبيقات والأدوات</h2><ul><li><a href="https://archivebox.io/">خادم ArchiveBox</a><a aria-label="خادم ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox لنظامَي macOS وiOS</a><a aria-label="ArchiveBox لنظامَي macOS وiOS — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox لنظام Android</a><a aria-label="ArchiveBox لنظام Android — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox لسطح المكتب</a><a aria-label="ArchiveBox لسطح المكتب — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">إضافة ArchiveBox للمتصفح</a><a aria-label="إضافة ArchiveBox للمتصفح — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">مكتبة إضافات ArchiveBox</a><a aria-label="مكتبة إضافات ArchiveBox — الشيفرة المصدرية على GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="عرض الشيفرة المصدرية على GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-Line truncated
<!-- /ARCHIVEBOX:FOOTER-START -->
<div class="abx-footer-local"><p>أرشفة ويب مفتوحة المصدر ومستضافة ذاتيًا. بترخيص MIT وبدعم غير ربحي ومصممة لمجموعات تدوم طويلًا.</p>
<div>
+6 -6
View File
@@ -158,7 +158,7 @@
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
<h3>Guarda pestañas en segundos, las almacena durante décadas</h3>
<p>Recopile marcadores para leer más tarde o conserve pruebas para casos legales con capturas verificables de nuestra <a href="https://tlsnotary.zervice.io/">Complemento de firma TLSNotary</a>.</p>
<p>Recopile marcadores para leer más tarde o conserve pruebas para casos legales con capturas verificables de nuestra <a href="https://tlsnotary.archivebox.io/">Complemento de firma TLSNotary</a>.</p>
</article>
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
@@ -204,7 +204,7 @@
<ul>
<li>Almacene capturas de pantalla, archivos PDF, encabezados, archivos WARC y extracción de texto.</li>
<li>Etiqueta y revisa fuentes a través de la interfaz web autohospedada.</li>
<li>Utilice pruebas ZK con <a href="https://tlsnotary.zervice.io/">TLSNotario</a> para demostrar la autenticidad del contenido.</li>
<li>Utilice pruebas ZK con <a href="https://tlsnotary.archivebox.io/">TLSNotario</a> para demostrar la autenticidad del contenido.</li>
</ul>
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="Salida de TLSNotary que verifica una respuesta archivada en la página de detalles de la instantánea" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
</article>
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Servidor doméstico</span></div></div><figcaption>En tu propio hardware</figcaption></figure>
<div class="command-card">
<div class="step"><span>1</span><div><h4>Elige tu plataforma</h4><p>Las integraciones comunitarias pueden ir por detrás de las versiones de Docker y Python.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">sin asaltar</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Instalar y abrir ArchiveBox</h4><p>Siga la guía de instalación de la plataforma, configure el almacenamiento persistente y luego abra la aplicación para finalizar la configuración.</p><p>Para TrueNAS, consulte la <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">Notas de la plataforma README</a> antes de elegir una integración.</p></div></div>
<div class="step"><span>1</span><div><h4>Elige tu plataforma</h4><p>Las integraciones comunitarias pueden ir por detrás de las versiones de Docker y Python.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">sin asaltar</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Instalar y abrir ArchiveBox</h4><p>Siga la guía de instalación de la plataforma, configure el almacenamiento persistente y luego abra la aplicación para finalizar la configuración.</p></div></div>
</div>
</div>
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<li><a aria-label="Complemento de archivo único" href="https://plugins.archivebox.io/#singlefile" title="Archivo único"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
<li><a aria-label="Complemento YouTube/yt-dlp" href="https://plugins.archivebox.io/#ytdlp" title="YouTube/yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
<li><a aria-label="Escala de cola" href="https://tailscale.com/" title="Escala de cola"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
<li><a aria-label="TLSNotario" href="https://tlsnotary.zervice.io/" title="TLSNotario"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
<li><a aria-label="TLSNotario" href="https://tlsnotary.archivebox.io/" title="TLSNotario"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
</ul>
<p class="configuration-description">ArchiveBox se puede configurar a través de la interfaz de usuario de administración, CLI, <code>ArchiveBox.conf</code>, o importando configuraciones y cookies desde Chrome u otros navegadores. El mismo modelo de configuración funciona en Docker, Docker Compose, instalaciones básicas, trabajos programados y ejecuciones CLI únicas.</p>
</div>
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<!-- ARCHIVEBOX:FOOTER-START -->
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">Herramientas de código abierto para conservar una copia duradera de la web.<br/>Tus datos. Tus dispositivos. Tu archivo.</p></div><p class="abx-footer-motto">Un hogar para la web<br/>que quieres conservar.</p></div>
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Aplicaciones y herramientas</h2><ul><li><a href="https://archivebox.io/">Servidor ArchiveBox</a><a aria-label="Servidor ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox para macOS e iOS</a><a aria-label="ArchiveBox para macOS e iOS — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox para Android</a><a aria-label="ArchiveBox para Android — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox para escritorio</a><a aria-label="ArchiveBox para escritorio — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extensión de navegador ArchiveBox</a><a aria-label="Extensión de navegador ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Biblioteca de complementos ArchiveBox</a><a aria-label="Biblioteca de complementos ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2Line truncated
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Aplicaciones y herramientas</h2><ul><li><a href="https://archivebox.io/">Servidor ArchiveBox</a><a aria-label="Servidor ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox para macOS e iOS</a><a aria-label="ArchiveBox para macOS e iOS — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox para Android</a><a aria-label="ArchiveBox para Android — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox para escritorio</a><a aria-label="ArchiveBox para escritorio — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extensión de navegador ArchiveBox</a><a aria-label="Extensión de navegador ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Biblioteca de complementos ArchiveBox</a><a aria-label="Biblioteca de complementos ArchiveBox — código fuente en GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Ver código fuente en GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2Line truncated
<!-- /ARCHIVEBOX:FOOTER-START -->
<div class="abx-footer-local"><p>Archivado web de código abierto y autoalojado. Con licencia MIT, apoyo de organizaciones sin ánimo de lucro y diseñado para colecciones duraderas.</p>
<div>
+6 -6
View File
@@ -158,7 +158,7 @@
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
<h3>Enregistre les onglets en quelques secondes, les stocke pendant des décennies</h3>
<p>Collectez des signets pour les lire plus tard ou conservez des preuves pour des affaires juridiques avec des captures vérifiables de notre <a href="https://tlsnotary.zervice.io/">Plugin de signature TLSNotary</a>.</p>
<p>Collectez des signets pour les lire plus tard ou conservez des preuves pour des affaires juridiques avec des captures vérifiables de notre <a href="https://tlsnotary.archivebox.io/">Plugin de signature TLSNotary</a>.</p>
</article>
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
@@ -204,7 +204,7 @@
<ul>
<li>Stockez des captures d'écran, des PDF, des en-têtes, des fichiers WARC et des extractions de texte.</li>
<li>Marquez et examinez les sources via l’interface Web auto-hébergée.</li>
<li>Utilisez des preuves ZK avec <a href="https://tlsnotary.zervice.io/">TLSNotaire</a> pour prouver l'authenticité du contenu.</li>
<li>Utilisez des preuves ZK avec <a href="https://tlsnotary.archivebox.io/">TLSNotaire</a> pour prouver l'authenticité du contenu.</li>
</ul>
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="Sortie TLSNotary vérifiant une réponse archivée sur la page de détails de l'instantané" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
</article>
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Serveur domestique</span></div></div><figcaption>Sur votre propre matériel</figcaption></figure>
<div class="command-card">
<div class="step"><span>1</span><div><h4>Choisissez votre plateforme</h4><p>Les intégrations communautaires peuvent être en retard par rapport aux versions Docker et Python.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">Sans raid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Installer et ouvrir ArchiveBox</h4><p>Suivez le guide d'installation de la plateforme, configurez le stockage persistant, puis ouvrez l'application pour terminer la configuration.</p><p>Pour TrueNAS, consultez le <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">Notes de la plateforme README</a> avant de choisir une intégration.</p></div></div>
<div class="step"><span>1</span><div><h4>Choisissez votre plateforme</h4><p>Les intégrations communautaires peuvent être en retard par rapport aux versions Docker et Python.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">Sans raid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Installer et ouvrir ArchiveBox</h4><p>Suivez le guide d'installation de la plateforme, configurez le stockage persistant, puis ouvrez l'application pour terminer la configuration.</p></div></div>
</div>
</div>
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<li><a aria-label="Plugin SingleFile" href="https://plugins.archivebox.io/#singlefile" title="Fichier unique"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
<li><a aria-label="Plugin YouTube/yt-dlp" href="https://plugins.archivebox.io/#ytdlp" title="YouTube / yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
<li><a aria-label="Échelle arrière" href="https://tailscale.com/" title="Échelle arrière"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
<li><a aria-label="TLSNotaire" href="https://tlsnotary.zervice.io/" title="TLSNotaire"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
<li><a aria-label="TLSNotaire" href="https://tlsnotary.archivebox.io/" title="TLSNotaire"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
</ul>
<p class="configuration-description">ArchiveBox peut être configuré via l'interface utilisateur d'administration, la CLI, <code>ArchiveBox.conf</code>, ou en important la configuration et les cookies depuis Chrome/d'autres navigateurs. Le même modèle de configuration fonctionne dans Docker, Docker Compose, les installations nues, les tâches planifiées et les exécutions CLI ponctuelles.</p>
</div>
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<!-- ARCHIVEBOX:FOOTER-START -->
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">Des outils open source pour conserver durablement une copie du Web.<br/>Vos données. Vos appareils. Vos archives.</p></div><p class="abx-footer-motto">Un espace pour le Web<br/>que vous souhaitez conserver.</p></div>
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Applications et outils</h2><ul><li><a href="https://archivebox.io/">Serveur ArchiveBox</a><a aria-label="Serveur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox pour macOS et iOS</a><a aria-label="ArchiveBox pour macOS et iOS — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox pour Android</a><a aria-label="ArchiveBox pour Android — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox pour ordinateur</a><a aria-label="ArchiveBox pour ordinateur — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extension de navigateur ArchiveBox</a><a aria-label="Extension de navigateur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Bibliothèque de plugins ArchiveBox</a><a aria-label="Bibliothèque de plugins ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.9Line truncated
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Applications et outils</h2><ul><li><a href="https://archivebox.io/">Serveur ArchiveBox</a><a aria-label="Serveur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox pour macOS et iOS</a><a aria-label="ArchiveBox pour macOS et iOS — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox pour Android</a><a aria-label="ArchiveBox pour Android — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox pour ordinateur</a><a aria-label="ArchiveBox pour ordinateur — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Extension de navigateur ArchiveBox</a><a aria-label="Extension de navigateur ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Bibliothèque de plugins ArchiveBox</a><a aria-label="Bibliothèque de plugins ArchiveBox — code source sur GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Voir le code source sur GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.9Line truncated
<!-- /ARCHIVEBOX:FOOTER-START -->
<div class="abx-footer-local"><p>Archivage web open source et auto-hébergé. Sous licence MIT, soutenu par des organismes à but non lucratif et conçu pour des collections durables.</p>
<div>
+5 -5
View File
@@ -149,7 +149,7 @@
<article>
<span class="card-icon" aria-hidden="true"><svg viewBox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"/></svg></span>
<h3>Saves tabs in seconds, stores them for decades</h3>
<p>Collect bookmarks to read later, or preserve evidence for legal cases with verifiable captures from our <a href="https://tlsnotary.zervice.io/">TLSNotary signing plugin</a>.</p>
<p>Collect bookmarks to read later, or preserve evidence for legal cases with verifiable captures from our <a href="https://tlsnotary.archivebox.io/">TLSNotary signing plugin</a>.</p>
</article>
<article>
<span class="card-icon" aria-hidden="true"><svg viewBox="0 0 24 24"><circle cx="10" cy="10" r="7"/><path d="m15 15 6 6"/></svg></span>
@@ -196,7 +196,7 @@
<ul>
<li>Store screenshots, PDFs, headers, WARC files, and text extraction.</li>
<li>Tag and review sources through the self-hosted web interface.</li>
<li>Use ZK proofs with <a href="https://tlsnotary.zervice.io/">TLSNotary</a> to prove content authenticity.</li>
<li>Use ZK proofs with <a href="https://tlsnotary.archivebox.io/">TLSNotary</a> to prove content authenticity.</li>
</ul>
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600" height="1000" loading="lazy" alt="TLSNotary output verifying an archived response on the snapshot details page"></a></figure>
</article>
@@ -363,8 +363,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<div class="install-panel" id="install-platforms" aria-labelledby="install-tab-platforms">
<figure class="install-art install-art--server"><div class="install-art-stage"><svg class="install-server-rack" viewBox="0 0 160 180" aria-hidden="true"><rect x="18" y="12" width="124" height="156" rx="12"/><rect x="32" y="28" width="96" height="34" rx="5"/><rect x="32" y="73" width="96" height="34" rx="5"/><rect x="32" y="118" width="96" height="34" rx="5"/><path d="M47 45h43m-43 45h43m-43 45h43"/><circle cx="111" cy="45" r="4"/><circle cx="111" cy="90" r="4"/><circle cx="111" cy="135" r="4"/></svg><div class="install-art-tile"><img src="./assets/install/archivebox-server.png" alt="" class="" loading="lazy"><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Home server</span></div></div><figcaption>On your own hardware</figcaption></figure>
<div class="command-card">
<div class="step"><span>1</span><div><h4>Choose your platform</h4><p>Community integrations may lag behind the Docker and Python releases.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">Unraid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Install and open ArchiveBox</h4><p>Follow the platform’s installation guide, configure persistent storage, then open the app to finish setup.</p><p>For TrueNAS, see the <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">README’s platform notes</a> before choosing an integration.</p></div></div>
<div class="step"><span>1</span><div><h4>Choose your platform</h4><p>Community integrations may lag behind the Docker and Python releases.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">Unraid</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Install and open ArchiveBox</h4><p>Follow the platform’s installation guide, configure persistent storage, then open the app to finish setup.</p></div></div>
</div>
</div>
<div class="install-panel" id="install-hosting" aria-labelledby="install-tab-hosting">
@@ -402,7 +402,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<li><a href="https://plugins.archivebox.io/#singlefile" aria-label="SingleFile plugin" title="SingleFile"><img src="./assets/plugins/singlefile.png" width="44" height="44" loading="lazy" alt=""></a></li>
<li><a href="https://plugins.archivebox.io/#ytdlp" aria-label="YouTube / yt-dlp plugin" title="YouTube / yt-dlp"><img src="./assets/plugins/youtube.svg" width="44" height="44" loading="lazy" alt=""></a></li>
<li><a href="https://tailscale.com/" aria-label="Tailscale" title="Tailscale"><img src="./assets/plugins/tailscale.svg" width="44" height="44" loading="lazy" alt=""></a></li>
<li><a href="https://tlsnotary.zervice.io/" aria-label="TLSNotary" title="TLSNotary"><img src="./assets/plugins/tlsnotary.svg" width="44" height="44" loading="lazy" alt=""></a></li>
<li><a href="https://tlsnotary.archivebox.io/" aria-label="TLSNotary" title="TLSNotary"><img src="./assets/plugins/tlsnotary.svg" width="44" height="44" loading="lazy" alt=""></a></li>
</ul>
<p class="configuration-description">ArchiveBox can be configured via the admin UI, CLI, <code>ArchiveBox.conf</code>, or by importing config and cookies from Chrome / other browsers. The same configuration model works in Docker, Docker Compose, bare-metal installs, scheduled jobs, and one-off CLI runs.</p>
</div>
+6 -6
View File
@@ -158,7 +158,7 @@
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
<h3>Сохраняет вкладки за считанные секунды и хранит их десятилетиями.</h3>
<p>Собирайте закладки, чтобы прочитать их позже, или сохраняйте доказательства по юридическим делам с помощью поддающихся проверке записей из наших <a href="https://tlsnotary.zervice.io/">Плагин подписи TLSNotary</a>.</p>
<p>Собирайте закладки, чтобы прочитать их позже, или сохраняйте доказательства по юридическим делам с помощью поддающихся проверке записей из наших <a href="https://tlsnotary.archivebox.io/">Плагин подписи TLSNotary</a>.</p>
</article>
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
@@ -204,7 +204,7 @@
<ul>
<li>Храните снимки экрана, PDF-файлы, заголовки, файлы WARC и извлечение текста.</li>
<li>Отмечайте и просматривайте источники через автономный веб-интерфейс.</li>
<li>Используйте доказательства ZK с <a href="https://tlsnotary.zervice.io/">TLSНотариус</a> для подтверждения подлинности контента.</li>
<li>Используйте доказательства ZK с <a href="https://tlsnotary.archivebox.io/">TLSНотариус</a> для подтверждения подлинности контента.</li>
</ul>
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="Вывод TLSNotary, подтверждающий заархивированный ответ на странице сведений о снимке" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
</article>
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>Домашний сервер</span></div></div><figcaption>На вашем собственном оборудовании</figcaption></figure>
<div class="command-card">
<div class="step"><span>1</span><div><h4>Выберите свою платформу</h4><p>Интеграция сообщества может отставать от выпусков Docker и Python.</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">Не набег</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Установите и откройте ArchiveBox.</h4><p>Следуйте инструкциям по установке платформы, настройте постоянное хранилище, затем откройте приложение, чтобы завершить настройку.</p><p>Информацию о TrueNAS см. <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">Примечания к платформе README</a> прежде чем выбрать интеграцию.</p></div></div>
<div class="step"><span>1</span><div><h4>Выберите свою платформу</h4><p>Интеграция сообщества может отставать от выпусков Docker и Python.</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">Не набег</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>Установите и откройте ArchiveBox.</h4><p>Следуйте инструкциям по установке платформы, настройте постоянное хранилище, затем откройте приложение, чтобы завершить настройку.</p></div></div>
</div>
</div>
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<li><a aria-label="Плагин SingleFile" href="https://plugins.archivebox.io/#singlefile" title="Один файл"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
<li><a aria-label="Плагин YouTube/yt-dlp" href="https://plugins.archivebox.io/#ytdlp" title="YouTube / yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
<li><a aria-label="Хвостовая чешуя" href="https://tailscale.com/" title="Хвостовая чешуя"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
<li><a aria-label="TLSНотариус" href="https://tlsnotary.zervice.io/" title="TLSНотариус"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
<li><a aria-label="TLSНотариус" href="https://tlsnotary.archivebox.io/" title="TLSНотариус"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
</ul>
<p class="configuration-description">ArchiveBox можно настроить через пользовательский интерфейс администратора, CLI, <code>ArchiveBox.conf</code>или импортировав конфигурацию и файлы cookie из Chrome/других браузеров. Одна и та же модель конфигурации работает в Docker, Docker Compose, при установке на «голое железо», запланированных заданиях и одноразовых запусках CLI.</p>
</div>
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<!-- ARCHIVEBOX:FOOTER-START -->
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">Открытые инструменты для долговременного сохранения копий веб-страниц.<br/>Ваши данные. Ваши устройства. Ваш архив.</p></div><p class="abx-footer-motto">Место для веб-страниц<br/>которые вы хотите сохранить.</p></div>
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Приложения и инструменты</h2><ul><li><a href="https://archivebox.io/">Сервер ArchiveBox</a><a aria-label="Сервер ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox для macOS и iOS</a><a aria-label="ArchiveBox для macOS и iOS — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox для Android</a><a aria-label="ArchiveBox для Android — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox для компьютера</a><a aria-label="ArchiveBox для компьютера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Расширение ArchiveBox для браузера</a><a aria-label="Расширение ArchiveBox для браузера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Библиотека плагинов ArchiveBox</a><a aria-label="Библиотека плагинов ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.Line truncated
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>Приложения и инструменты</h2><ul><li><a href="https://archivebox.io/">Сервер ArchiveBox</a><a aria-label="Сервер ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">ArchiveBox для macOS и iOS</a><a aria-label="ArchiveBox для macOS и iOS — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">ArchiveBox для Android</a><a aria-label="ArchiveBox для Android — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox для компьютера</a><a aria-label="ArchiveBox для компьютера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">Расширение ArchiveBox для браузера</a><a aria-label="Расширение ArchiveBox для браузера — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">Библиотека плагинов ArchiveBox</a><a aria-label="Библиотека плагинов ArchiveBox — исходный код на GitHub" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="Смотреть исходный код на GitHub"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.Line truncated
<!-- /ARCHIVEBOX:FOOTER-START -->
<div class="abx-footer-local"><p>Открытое веб-архивирование на собственном сервере. Лицензия MIT, поддержка некоммерческих организаций и ориентация на долговременные коллекции.</p>
<div>
+6 -6
View File
@@ -158,7 +158,7 @@
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><path d="M6 3h12v18l-6-4-6 4V3Z"></path></svg></span>
<h3>在几秒钟内保存选项卡,将其存储数十年</h3>
<p>收集书签以供稍后阅读,或通过我们的可验证捕获保存法律案件的证据 <a href="https://tlsnotary.zervice.io/">TLSNotary 签名插件</a>.</p>
<p>收集书签以供稍后阅读,或通过我们的可验证捕获保存法律案件的证据 <a href="https://tlsnotary.archivebox.io/">TLSNotary 签名插件</a>.</p>
</article>
<article>
<span aria-hidden="true" class="card-icon"><svg viewbox="0 0 24 24"><circle cx="10" cy="10" r="7"></circle><path d="m15 15 6 6"></path></svg></span>
@@ -204,7 +204,7 @@
<ul>
<li>存储屏幕截图、PDF、标题、WARC 文件和文本提取。</li>
<li>通过自托管 Web 界面标记和审查源。</li>
<li>使用 ZK 证明 <a href="https://tlsnotary.zervice.io/">TLS公证人</a> 以证明内容的真实性。</li>
<li>使用 ZK 证明 <a href="https://tlsnotary.archivebox.io/">TLS公证人</a> 以证明内容的真实性。</li>
</ul>
<figure class="card-shot"><a href="/screenshots/snapshot-view-tlsnotary-desktop.png"><img alt="TLSNotary 输出验证快照详细信息页面上的存档响应" height="1000" loading="lazy" src="/screenshots/snapshot-view-tlsnotary-desktop.png" width="1600"/></a></figure>
</article>
@@ -370,8 +370,8 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<div aria-labelledby="install-tab-platforms" class="install-panel" id="install-platforms">
<figure class="install-art install-art--server"><div class="install-art-stage"><svg aria-hidden="true" class="install-server-rack" viewbox="0 0 160 180"><rect height="156" rx="12" width="124" x="18" y="12"></rect><rect height="34" rx="5" width="96" x="32" y="28"></rect><rect height="34" rx="5" width="96" x="32" y="73"></rect><rect height="34" rx="5" width="96" x="32" y="118"></rect><path d="M47 45h43m-43 45h43m-43 45h43"></path><circle cx="111" cy="45" r="4"></circle><circle cx="111" cy="90" r="4"></circle><circle cx="111" cy="135" r="4"></circle></svg><div class="install-art-tile"><img alt="" class="" loading="lazy" src="../assets/install/archivebox-server.png"/><span>ArchiveBox</span></div><div class="install-art-chips"><span>NAS</span><span>家庭服务器</span></div></div><figcaption>在您自己的硬件上</figcaption></figure>
<div class="command-card">
<div class="step"><span>1</span><div><h4>选择您的平台</h4><p>社区集成可能落后于 Docker 和 Python 版本。</p><div class="mini-links"><a href="https://unraid.net/community/apps?q=archivebox#r">解除突袭</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>安装并打开 ArchiveBox</h4><p>按照平台的安装指南,配置持久存储,然后打开应用程序完成设置。</p><p>对于 TrueNAS,请参阅 <a href="https://github.com/ArchiveBox/ArchiveBox#quickstart">README 的平台说明</a> 在选择集成之前。</p></div></div>
<div class="step"><span>1</span><div><h4>选择您的平台</h4><p>社区集成可能落后于 Docker 和 Python 版本。</p><div class="mini-links"><a href="https://apps.truenas.com/catalog/archivebox_community/">TrueNAS</a><a href="https://unraid.net/community/apps?q=archivebox#r">解除突袭</a><a href="https://community-scripts.github.io/ProxmoxVE/scripts?id=archivebox">Proxmox</a><a href="https://github.com/YunoHost-Apps/archivebox_ynh">YunoHost</a><a href="https://www.cloudron.io/store/io.archivebox.cloudronapp.html">Cloudron</a><a href="https://docs.saltbox.dev/sandbox/apps/archivebox/">Saltbox</a><a href="https://portainer-templates.as93.net/archivebox">Portainer</a><a href="https://runtipi.io/docs/apps-available">Runtipi</a></div></div></div>
<div class="step"><span>2</span><div><h4>安装并打开 ArchiveBox</h4><p>按照平台的安装指南,配置持久存储,然后打开应用程序完成设置。</p></div></div>
</div>
</div>
<div aria-labelledby="install-tab-hosting" class="install-panel" id="install-hosting">
@@ -408,7 +408,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<li><a aria-label="单文件插件" href="https://plugins.archivebox.io/#singlefile" title="单一文件"><img alt="" height="44" loading="lazy" src="../assets/plugins/singlefile.png" width="44"/></a></li>
<li><a aria-label="YouTube / yt-dlp 插件" href="https://plugins.archivebox.io/#ytdlp" title="YouTube / yt-dlp"><img alt="" height="44" loading="lazy" src="../assets/plugins/youtube.svg" width="44"/></a></li>
<li><a aria-label="尾鳞" href="https://tailscale.com/" title="尾鳞"><img alt="" height="44" loading="lazy" src="../assets/plugins/tailscale.svg" width="44"/></a></li>
<li><a aria-label="TLS公证人" href="https://tlsnotary.zervice.io/" title="TLS公证人"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
<li><a aria-label="TLS公证人" href="https://tlsnotary.archivebox.io/" title="TLS公证人"><img alt="" height="44" loading="lazy" src="../assets/plugins/tlsnotary.svg" width="44"/></a></li>
</ul>
<p class="configuration-description">ArchiveBox 可以通过管理 UI、CLI 进行配置, <code>ArchiveBox.conf</code>,或通过从 Chrome/其他浏览器导入配置和 cookie。相同的配置模型适用于 Docker、Docker Compose、裸机安装、计划作业和一次性 CLI 运行。</p>
</div>
@@ -627,7 +627,7 @@ archivebox server 0.0.0.0:5797</code></pre></div></div></div></div>
<!-- ARCHIVEBOX:FOOTER-START -->
<footer aria-label="ArchiveBox ecosystem" class="abx-footer">
<div class="abx-footer-intro"><div><a class="abx-brand" href="https://archivebox.io/"><svg aria-hidden="true" class="abx-logo" viewbox="0 0 40 40"><rect fill="#9b2854" height="40" rx="10" width="40"></rect><path d="M10 14h20v17H10z" fill="#fff4ed"></path><path d="M8 9h24v6H8z" fill="#fff4ed"></path><path d="M16 19h8v3h-8z" fill="#9b2854"></path><path d="M12 31V17" stroke="#9b2854" stroke-width="2"></path></svg><span>ArchiveBox</span></a><p class="abx-tagline">用于长期保存网页副本的开源工具。<br/>你的数据。你的设备。你的归档。</p></div><p class="abx-footer-motto">为网页安个家<br/>保存你想保留的内容。</p></div>
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>应用与工具</h2><ul><li><a href="https://archivebox.io/">ArchiveBox 服务器</a><a aria-label="ArchiveBox 服务器 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">macOS 和 iOS 版 ArchiveBox</a><a aria-label="macOS 和 iOS 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">Android 版 ArchiveBox</a><a aria-label="Android 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox 桌面版</a><a aria-label="ArchiveBox 桌面版 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox 浏览器扩展</a><a aria-label="ArchiveBox 浏览器扩展 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox 插件库</a><a aria-label="ArchiveBox 插件库 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z">Line truncated
<div class="abx-footer-grid"><section class="abx-footer-column"><h2>应用与工具</h2><ul><li><a href="https://archivebox.io/">ArchiveBox 服务器</a><a aria-label="ArchiveBox 服务器 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ArchiveBox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://app.archivebox.io/#get-started">macOS 和 iOS 版 ArchiveBox</a><a aria-label="macOS 和 iOS 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/ios-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://android.archivebox.io/">Android 版 ArchiveBox</a><a aria-label="Android 版 ArchiveBox — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/android-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://electron.archivebox.io/">ArchiveBox 桌面版</a><a aria-label="ArchiveBox 桌面版 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/electron-archivebox" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://extension.archivebox.io/">ArchiveBox 浏览器扩展</a><a aria-label="ArchiveBox 浏览器扩展 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/archivebox-browser-extension" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z"></path></svg></a></li><li><a href="https://plugins.archivebox.io/">ArchiveBox 插件库</a><a aria-label="ArchiveBox 插件库 — GitHub 源代码" class="abx-source" href="https://github.com/ArchiveBox/abx-plugins" title="在 GitHub 查看源代码"><svg aria-hidden="true" viewbox="0 0 24 24"><path d="M12 .75a11.25 11.25 0 0 0-3.56 21.92c.56.1.77-.24.77-.54v-2.1c-3.14.68-3.8-1.33-3.8-1.33-.51-1.3-1.25-1.65-1.25-1.65-1.02-.7.08-.69.08-.69 1.13.08 1.73 1.16 1.73 1.16 1 1.72 2.63 1.22 3.27.93.1-.73.39-1.22.71-1.5-2.5-.28-5.13-1.25-5.13-5.56 0-1.23.44-2.23 1.16-3.02-.12-.28-.5-1.43.11-2.98 0 0 .95-.3 3.1 1.15a10.8 10.8 0 0 1 5.64 0c2.15-1.46 3.1-1.15 3.1-1.15.62 1.55.23 2.7.12 2.98.72.79 1.16 1.8 1.16 3.02 0 4.32-2.64 5.27-5.15 5.55.4.35.76 1.03.76 2.08v3.11c0 .3.2.65.77.54A11.25 11.25 0 0 0 12 .75Z">Line truncated
<!-- /ARCHIVEBOX:FOOTER-START -->
<div class="abx-footer-local"><p>开源、自托管的网页归档。采用 MIT 许可证,由非营利组织支持,为长期保存的收藏而设计。</p>
<div>
+6 -6
View File
@@ -1,6 +1,6 @@
[project]
name = "archivebox"
version = "0.9.71rc3"
version = "0.9.71rc42"
requires-python = ">=3.13"
description = "Self-hosted internet archiving solution."
authors = [{name = "Nick Sweeting", email = "pyproject.toml@archivebox.io"}]
@@ -77,10 +77,10 @@ dependencies = [
"w3lib>=2.2.1", # used for parsing content-type encoding from http response headers & html tags
### Extractor dependencies (runtime binaries resolved through abxpkg)
### Binary/Package Management
"abxbus==2.5.65", # direct imports only; version constrained by abx-dl -> abx-plugins
"abxpkg==1.13.11", # direct imports only; version constrained by abx-dl -> abx-plugins
"abx-plugins[opencode]==1.13.82", # optional AI runtime clients are owned by the plugin
"abx-dl==1.13.59", # shared ArchiveBox downloader package
"abxbus==2.5.68", # direct imports only; version constrained by abx-dl -> abx-plugins
"abxpkg==1.13.15", # direct imports only; version constrained by abx-dl -> abx-plugins
"abx-plugins[opencode]==1.13.103", # optional AI runtime clients are owned by the plugin
"abx-dl==1.13.71", # shared ArchiveBox downloader package
### UUID7 backport for Python <3.14
"uuid7>=0.1.0; python_version < '3.14'", # provides the uuid_extensions module on Python 3.13
"browser-cookie3>=0.20.1",
@@ -343,7 +343,7 @@ Donate = "https://github.com/ArchiveBox/ArchiveBox/wiki/Donations"
[tool.bumpver]
current_version = "v0.9.71rc3"
current_version = "v0.9.71rc42"
version_pattern = "vMAJOR.MINOR.PATCH[PYTAGNUM]"
commit_message = "bump version {old_version} -> {new_version}"
tag_message = "{new_version}"
+2 -4
View File
@@ -68,9 +68,7 @@ archivebox run
## Verification
```bash
uv run pytest archivebox/tests/test_cli_add.py::test_add_help_shows_depth_and_tag_options -q
uv run prek run --all-files
```
Run the test suite with `uv run pytest archivebox/tests -x` and repository checks
with `uv run prek run --all-files`. For focused work, select the relevant tests.
Releases are published only by `.github/workflows/release.yml` after the complete `dev` CI workflow succeeds. Local development and deployment commands must not publish packages, images, tags, or GitHub releases.
Generated
+19 -19
View File
@@ -13,18 +13,18 @@ supported-markers = [
]
[options]
exclude-newer = "2026-09-22T07:00:25.284838761Z"
exclude-newer = "2026-09-23T08:33:14.714060607Z"
exclude-newer-span = "P5D"
[options.exclude-newer-package]
abxbus = { timestamp = "2026-09-27T07:00:24.284850353Z", span = "PT1S" }
abxbus = { timestamp = "2026-09-28T08:33:13.714073511Z", span = "PT1S" }
abx-plugins = "2100-01-01T00:00:00Z"
abx-dl = "2100-01-01T00:00:00Z"
abxpkg = "2100-01-01T00:00:00Z"
[[package]]
name = "abx-dl"
version = "1.13.59"
version = "1.13.71"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "abx-plugins", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -38,14 +38,14 @@ dependencies = [
{ name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8c/e3/d4e9459cb2ad0ac0a37e4a0747bd3b0bab414294706e624a32fd0de19800/abx_dl-1.13.59.tar.gz", hash = "sha256:0c431e6e0dbc088ecc1625843405b89093be8e99c1e72672d9266e8cfbc41fb7", size = 97393 }
sdist = { url = "https://files.pythonhosted.org/packages/79/67/9eaf667a429db72f0609b493aca107ff42f29a31dfd2333fcea02ece900a/abx_dl-1.13.71.tar.gz", hash = "sha256:1fdb6e584e2d4e7d7c04048cbd5c37bef3578352d6c2fe7ab76fc706ddadf09b", size = 97387 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/97/16/024bcbddc7c75fc3adbfdd7593374add9c95f46d76f55b2209185e2702f4/abx_dl-1.13.59-py3-none-any.whl", hash = "sha256:9bbce2ceebcbfe98473beb4d71d1aecfb1c8364a5fef3fef46f5cd2150f939d9", size = 102837 },
{ url = "https://files.pythonhosted.org/packages/31/23/d12ed879d5253ae6bbbf0e2ced77b0b59f3b4d4c3b381cc84518fee0443e/abx_dl-1.13.71-py3-none-any.whl", hash = "sha256:0585182839d8e9d3c93ee87d0ca6e2fa2d9d8ac936515f2700046324d210ce29", size = 102837 },
]
[[package]]
name = "abx-plugins"
version = "1.13.82"
version = "1.13.103"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "abxbus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -54,9 +54,9 @@ dependencies = [
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "uv", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ec/6f/c13daea79627fdd589aa2a932ecc9d82251229e1d836b76bc658d0f4b092/abx_plugins-1.13.82.tar.gz", hash = "sha256:64916aebd3db8aeb131f69ac6c6fa9b9b436944e174104a1e6652556cf4f43bb", size = 458681, upload-time = "2026-09-27T06:43:53.582Z" }
sdist = { url = "https://files.pythonhosted.org/packages/5c/ba/9b9f6d72c95026fbaf8721adc36705a5f3f6916387d2c7d15d29f095e4e0/abx_plugins-1.13.103.tar.gz", hash = "sha256:ff49efbee93a54203ecf3680251ef60299f29d4c77b1d20bfbfebfabdccd93b2", size = 460937, upload-time = "2026-09-28T07:58:11.436Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/bb/82/fb926f89622031f8611b942ca9855d42a3158a7640518b2e1d8cac05b4fb/abx_plugins-1.13.82-py3-none-any.whl", hash = "sha256:09a7a7f57beb2b34106765c79ca25d9b27f1dc661d8d16dc2ef6b5bb085c8691", size = 668544, upload-time = "2026-09-27T06:43:51.765Z" },
{ url = "https://files.pythonhosted.org/packages/69/ec/1c2db942f2ebd8945160084aaed8e4cac29a3364937c64771c8e76700cef/abx_plugins-1.13.103-py3-none-any.whl", hash = "sha256:54ee14dbad090f180b7d35075c175460be0a783745ed86a732bb960157778dff", size = 673577, upload-time = "2026-09-28T07:58:09.949Z" },
]
[package.optional-dependencies]
@@ -68,21 +68,21 @@ opencode = [
[[package]]
name = "abxbus"
version = "2.5.65"
version = "2.5.68"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "uuid7", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/6b/87/389195fe7d4c864cc1866f8a2b8292ec39c23200fb5f865985d24ad7a56f/abxbus-2.5.65.tar.gz", hash = "sha256:7edbb6036a1c01f13ea5c61f746c54233e709701e53cb674b3159d3727611ef2", size = 134216, upload-time = "2026-09-23T19:18:02.517Z" }
sdist = { url = "https://files.pythonhosted.org/packages/8e/d7/0c92e436230b40ee4c21c83e4081ab67a6930538e3ff759857c3d4cebe51/abxbus-2.5.68.tar.gz", hash = "sha256:e53c68467749f17ba14b982184b02aed8ba106e36412217f1c1cd61d8ce5a7b0", size = 134319, upload-time = "2026-09-28T07:13:42.183Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d3/21/a1b144425294d71ae6d3874dc27bceb0e62c5fc15d72ca15bab11e0dc724/abxbus-2.5.65-py3-none-any.whl", hash = "sha256:8e9cc083f84ba190e843a3dad13b333bdb857204a2db7e0411d3574d45861131", size = 127915, upload-time = "2026-09-23T19:18:01.182Z" },
{ url = "https://files.pythonhosted.org/packages/bc/ee/e2b567d576b13bf32aaddcacf376a89c4a4df0de1081f95a56a0d03bdc23/abxbus-2.5.68-py3-none-any.whl", hash = "sha256:61f3ba656f1010b6801d8a625f65f124521e6ec2e03f454baf7491e5ed4fb345", size = 127969, upload-time = "2026-09-28T07:13:40.627Z" },
]
[[package]]
name = "abxpkg"
version = "1.13.11"
version = "1.13.15"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -90,9 +90,9 @@ dependencies = [
{ name = "rich-click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ef/6f/b09edc9b41266d7be1433891ac7486dbc656487c77a2e3b210099a18f2ee/abxpkg-1.13.11.tar.gz", hash = "sha256:95a9deb431b8b755e88bcef37f67c69fb045fd0416fe8da70bcfdce7053913d4", size = 242757, upload-time = "2026-09-26T03:08:51.1Z" }
sdist = { url = "https://files.pythonhosted.org/packages/62/34/16a0501195c9a29c5f532a23a07e50237d0e492b78b5d9d8f43c13a7517c/abxpkg-1.13.15.tar.gz", hash = "sha256:7f29ca4432399ab15180bd58a60f39a3ef7fedcd7940affe2eda6b57e914c98c", size = 242799, upload-time = "2026-09-28T07:26:27.62Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/db/d9/54a2df40e69c451f82e1504e22d272b935a9dd1fe4c3d6d971cac59dab8b/abxpkg-1.13.11-py3-none-any.whl", hash = "sha256:23797f0e6a25012d370517fa8253ef0c898452591e0ad37bfdcd0ea5861c89d3", size = 254167, upload-time = "2026-09-26T03:08:52.578Z" },
{ url = "https://files.pythonhosted.org/packages/7d/85/988013c6e773bed15f1671e017317edc3b047effa594cbf826af49abce11/abxpkg-1.13.15-py3-none-any.whl", hash = "sha256:8b1924225c4f79408d2d1729c59614014779fbae2800d1e7e2ce3ef1389be198", size = 254220, upload-time = "2026-09-28T07:26:26.22Z" },
]
[[package]]
@@ -127,7 +127,7 @@ wheels = [
[[package]]
name = "archivebox"
version = "0.9.71rc3"
version = "0.9.71rc42"
source = { editable = "." }
dependencies = [
{ name = "abx-dl", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -227,10 +227,10 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "abx-dl", specifier = "==1.13.59" },
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.82" },
{ name = "abxbus", specifier = "==2.5.65" },
{ name = "abxpkg", specifier = "==1.13.11" },
{ name = "abx-dl", specifier = "==1.13.71" },
{ name = "abx-plugins", extras = ["opencode"], specifier = "==1.13.103" },
{ name = "abxbus", specifier = "==2.5.68" },
{ name = "abxpkg", specifier = "==1.13.15" },
{ name = "archivebox", extras = ["sonic", "ldap", "debug"], marker = "extra == 'all'" },
{ name = "atomicwrites", specifier = "==1.4.1" },
{ name = "base32-crockford", specifier = ">=0.3.0" },